Search in sources :

Example 1 with WorkerTaskRunnerQueryAdapter

use of org.apache.druid.indexing.overlord.WorkerTaskRunnerQueryAdapter in project druid by druid-io.

the class OverlordTest method testOverlordRun.

@Test(timeout = 60_000L)
public void testOverlordRun() throws Exception {
    // basic task master lifecycle test
    taskMaster.start();
    announcementLatch.await();
    while (!taskMaster.isLeader()) {
        // I believe the control will never reach here and thread will never sleep but just to be on safe side
        Thread.sleep(10);
    }
    Assert.assertEquals(taskMaster.getCurrentLeader(), druidNode.getHostAndPort());
    final TaskStorageQueryAdapter taskStorageQueryAdapter = new TaskStorageQueryAdapter(taskStorage, taskLockbox);
    final WorkerTaskRunnerQueryAdapter workerTaskRunnerQueryAdapter = new WorkerTaskRunnerQueryAdapter(taskMaster, null);
    // Test Overlord resource stuff
    overlordResource = new OverlordResource(taskMaster, taskStorageQueryAdapter, new IndexerMetadataStorageAdapter(taskStorageQueryAdapter, null), null, null, null, AuthTestUtils.TEST_AUTHORIZER_MAPPER, workerTaskRunnerQueryAdapter, null);
    Response response = overlordResource.getLeader();
    Assert.assertEquals(druidNode.getHostAndPort(), response.getEntity());
    final String taskId_0 = "0";
    NoopTask task_0 = NoopTask.create(taskId_0, 0);
    response = overlordResource.taskPost(task_0, req);
    Assert.assertEquals(200, response.getStatus());
    Assert.assertEquals(ImmutableMap.of("task", taskId_0), response.getEntity());
    // Duplicate task - should fail
    response = overlordResource.taskPost(task_0, req);
    Assert.assertEquals(400, response.getStatus());
    // Task payload for task_0 should be present in taskStorage
    response = overlordResource.getTaskPayload(taskId_0);
    Assert.assertEquals(task_0, ((TaskPayloadResponse) response.getEntity()).getPayload());
    // Task not present in taskStorage - should fail
    response = overlordResource.getTaskPayload("whatever");
    Assert.assertEquals(404, response.getStatus());
    // Task status of the submitted task should be running
    response = overlordResource.getTaskStatus(taskId_0);
    Assert.assertEquals(taskId_0, ((TaskStatusResponse) response.getEntity()).getTask());
    Assert.assertEquals(TaskStatus.running(taskId_0).getStatusCode(), ((TaskStatusResponse) response.getEntity()).getStatus().getStatusCode());
    // Simulate completion of task_0
    taskCompletionCountDownLatches[Integer.parseInt(taskId_0)].countDown();
    // Wait for taskQueue to handle success status of task_0
    waitForTaskStatus(taskId_0, TaskState.SUCCESS);
    // Manually insert task in taskStorage
    // Verifies sync from storage
    final String taskId_1 = "1";
    NoopTask task_1 = NoopTask.create(taskId_1, 0);
    taskStorage.insert(task_1, TaskStatus.running(taskId_1));
    // Wait for task runner to run task_1
    runTaskCountDownLatches[Integer.parseInt(taskId_1)].await();
    response = overlordResource.getRunningTasks(null, req);
    // 1 task that was manually inserted should be in running state
    Assert.assertEquals(1, (((List) response.getEntity()).size()));
    final TaskStatusPlus taskResponseObject = ((List<TaskStatusPlus>) response.getEntity()).get(0);
    Assert.assertEquals(taskId_1, taskResponseObject.getId());
    Assert.assertEquals(TASK_LOCATION, taskResponseObject.getLocation());
    // Simulate completion of task_1
    taskCompletionCountDownLatches[Integer.parseInt(taskId_1)].countDown();
    // Wait for taskQueue to handle success status of task_1
    waitForTaskStatus(taskId_1, TaskState.SUCCESS);
    // should return number of tasks which are not in running state
    response = overlordResource.getCompleteTasks(null, req);
    Assert.assertEquals(2, (((List) response.getEntity()).size()));
    response = overlordResource.getCompleteTasks(1, req);
    Assert.assertEquals(1, (((List) response.getEntity()).size()));
    taskMaster.stop();
    Assert.assertFalse(taskMaster.isLeader());
    EasyMock.verify(taskLockbox, taskActionClientFactory);
}
Also used : IndexerMetadataStorageAdapter(org.apache.druid.indexing.overlord.IndexerMetadataStorageAdapter) Response(javax.ws.rs.core.Response) TaskStatusPlus(org.apache.druid.indexer.TaskStatusPlus) List(java.util.List) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) NoopTask(org.apache.druid.indexing.common.task.NoopTask) WorkerTaskRunnerQueryAdapter(org.apache.druid.indexing.overlord.WorkerTaskRunnerQueryAdapter) TaskStorageQueryAdapter(org.apache.druid.indexing.overlord.TaskStorageQueryAdapter) Test(org.junit.Test)

Example 2 with WorkerTaskRunnerQueryAdapter

use of org.apache.druid.indexing.overlord.WorkerTaskRunnerQueryAdapter in project druid by druid-io.

the class OverlordResourceTest method setUp.

@Before
public void setUp() {
    taskRunner = EasyMock.createMock(TaskRunner.class);
    configManager = EasyMock.createMock(JacksonConfigManager.class);
    provisioningStrategy = EasyMock.createMock(ProvisioningStrategy.class);
    taskMaster = EasyMock.createStrictMock(TaskMaster.class);
    taskStorageQueryAdapter = EasyMock.createStrictMock(TaskStorageQueryAdapter.class);
    indexerMetadataStorageAdapter = EasyMock.createStrictMock(IndexerMetadataStorageAdapter.class);
    req = EasyMock.createStrictMock(HttpServletRequest.class);
    workerTaskRunnerQueryAdapter = EasyMock.createStrictMock(WorkerTaskRunnerQueryAdapter.class);
    EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.of(taskRunner)).anyTimes();
    AuthorizerMapper authMapper = new AuthorizerMapper(null) {

        @Override
        public Authorizer getAuthorizer(String name) {
            return new Authorizer() {

                @Override
                public Access authorize(AuthenticationResult authenticationResult, Resource resource, Action action) {
                    final String username = authenticationResult.getIdentity();
                    switch(resource.getName()) {
                        case "allow":
                            return new Access(true);
                        case Datasources.WIKIPEDIA:
                            // Only "Wiki Reader" can read "wikipedia"
                            return new Access(action == Action.READ && Users.WIKI_READER.equals(username));
                        case Datasources.BUZZFEED:
                            // Only "Buzz Reader" can read "buzzfeed"
                            return new Access(action == Action.READ && Users.BUZZ_READER.equals(username));
                        default:
                            return new Access(false);
                    }
                }
            };
        }
    };
    overlordResource = new OverlordResource(taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, null, configManager, null, authMapper, workerTaskRunnerQueryAdapter, provisioningStrategy);
}
Also used : IndexerMetadataStorageAdapter(org.apache.druid.indexing.overlord.IndexerMetadataStorageAdapter) Action(org.apache.druid.server.security.Action) JacksonConfigManager(org.apache.druid.common.config.JacksonConfigManager) Resource(org.apache.druid.server.security.Resource) Access(org.apache.druid.server.security.Access) TaskStorageQueryAdapter(org.apache.druid.indexing.overlord.TaskStorageQueryAdapter) TaskRunner(org.apache.druid.indexing.overlord.TaskRunner) WorkerTaskRunner(org.apache.druid.indexing.overlord.WorkerTaskRunner) ProvisioningStrategy(org.apache.druid.indexing.overlord.autoscaling.ProvisioningStrategy) AuthenticationResult(org.apache.druid.server.security.AuthenticationResult) HttpServletRequest(javax.servlet.http.HttpServletRequest) Authorizer(org.apache.druid.server.security.Authorizer) AuthorizerMapper(org.apache.druid.server.security.AuthorizerMapper) TaskMaster(org.apache.druid.indexing.overlord.TaskMaster) WorkerTaskRunnerQueryAdapter(org.apache.druid.indexing.overlord.WorkerTaskRunnerQueryAdapter) Before(org.junit.Before)

Aggregations

IndexerMetadataStorageAdapter (org.apache.druid.indexing.overlord.IndexerMetadataStorageAdapter)2 TaskStorageQueryAdapter (org.apache.druid.indexing.overlord.TaskStorageQueryAdapter)2 WorkerTaskRunnerQueryAdapter (org.apache.druid.indexing.overlord.WorkerTaskRunnerQueryAdapter)2 ImmutableList (com.google.common.collect.ImmutableList)1 ArrayList (java.util.ArrayList)1 List (java.util.List)1 HttpServletRequest (javax.servlet.http.HttpServletRequest)1 Response (javax.ws.rs.core.Response)1 JacksonConfigManager (org.apache.druid.common.config.JacksonConfigManager)1 TaskStatusPlus (org.apache.druid.indexer.TaskStatusPlus)1 NoopTask (org.apache.druid.indexing.common.task.NoopTask)1 TaskMaster (org.apache.druid.indexing.overlord.TaskMaster)1 TaskRunner (org.apache.druid.indexing.overlord.TaskRunner)1 WorkerTaskRunner (org.apache.druid.indexing.overlord.WorkerTaskRunner)1 ProvisioningStrategy (org.apache.druid.indexing.overlord.autoscaling.ProvisioningStrategy)1 Access (org.apache.druid.server.security.Access)1 Action (org.apache.druid.server.security.Action)1 AuthenticationResult (org.apache.druid.server.security.AuthenticationResult)1 Authorizer (org.apache.druid.server.security.Authorizer)1 AuthorizerMapper (org.apache.druid.server.security.AuthorizerMapper)1