Search in sources :

Example 6 with TaskRunner

use of org.apache.druid.indexing.overlord.TaskRunner in project druid by druid-io.

the class OverlordResource method getTaskStatus.

@GET
@Path("/task/{taskid}/status")
@Produces(MediaType.APPLICATION_JSON)
@ResourceFilters(TaskResourceFilter.class)
public Response getTaskStatus(@PathParam("taskid") String taskid) {
    final TaskInfo<Task, TaskStatus> taskInfo = taskStorageQueryAdapter.getTaskInfo(taskid);
    TaskStatusResponse response = null;
    if (taskInfo != null) {
        if (taskMaster.getTaskRunner().isPresent()) {
            final TaskRunner taskRunner = taskMaster.getTaskRunner().get();
            final TaskRunnerWorkItem workItem = taskRunner.getKnownTasks().stream().filter(item -> item.getTaskId().equals(taskid)).findAny().orElse(null);
            if (workItem != null) {
                response = new TaskStatusResponse(workItem.getTaskId(), new TaskStatusPlus(taskInfo.getId(), taskInfo.getTask() == null ? null : taskInfo.getTask().getGroupId(), taskInfo.getTask() == null ? null : taskInfo.getTask().getType(), taskInfo.getCreatedTime(), // TaskStorage API doesn't yet allow it.
                DateTimes.EPOCH, taskInfo.getStatus().getStatusCode(), taskRunner.getRunnerTaskState(workItem.getTaskId()), taskInfo.getStatus().getDuration(), workItem.getLocation(), taskInfo.getDataSource(), taskInfo.getStatus().getErrorMsg()));
            }
        }
        if (response == null) {
            response = new TaskStatusResponse(taskid, new TaskStatusPlus(taskInfo.getId(), taskInfo.getTask() == null ? null : taskInfo.getTask().getGroupId(), taskInfo.getTask() == null ? null : taskInfo.getTask().getType(), taskInfo.getCreatedTime(), // TaskStorage API doesn't yet allow it.
            DateTimes.EPOCH, taskInfo.getStatus().getStatusCode(), RunnerTaskState.WAITING, taskInfo.getStatus().getDuration(), taskInfo.getStatus().getLocation() == null ? TaskLocation.unknown() : taskInfo.getStatus().getLocation(), taskInfo.getDataSource(), taskInfo.getStatus().getErrorMsg()));
        }
    } else {
        response = new TaskStatusResponse(taskid, null);
    }
    final Response.Status status = response.getStatus() == null ? Response.Status.NOT_FOUND : Response.Status.OK;
    return Response.status(status).entity(response).build();
}
Also used : Response(javax.ws.rs.core.Response) Task(org.apache.druid.indexing.common.task.Task) TaskRunnerWorkItem(org.apache.druid.indexing.overlord.TaskRunnerWorkItem) Status(javax.ws.rs.core.Response.Status) TaskStatusPlus(org.apache.druid.indexer.TaskStatusPlus) TaskStatus(org.apache.druid.indexer.TaskStatus) TaskRunner(org.apache.druid.indexing.overlord.TaskRunner) WorkerTaskRunner(org.apache.druid.indexing.overlord.WorkerTaskRunner) Path(javax.ws.rs.Path) ResourceFilters(com.sun.jersey.spi.container.ResourceFilters) Produces(javax.ws.rs.Produces) GET(javax.ws.rs.GET)

Example 7 with TaskRunner

use of org.apache.druid.indexing.overlord.TaskRunner in project druid by druid-io.

the class SeekableStreamSupervisorStateTest method setupTest.

@Before
public void setupTest() {
    taskStorage = createMock(TaskStorage.class);
    taskMaster = createMock(TaskMaster.class);
    taskRunner = createMock(TaskRunner.class);
    taskQueue = createMock(TaskQueue.class);
    indexerMetadataStorageCoordinator = createMock(IndexerMetadataStorageCoordinator.class);
    taskClientFactory = createMock(SeekableStreamIndexTaskClientFactory.class);
    spec = createMock(SeekableStreamSupervisorSpec.class);
    indexTaskClient = createMock(SeekableStreamIndexTaskClient.class);
    recordSupplier = (RecordSupplier<String, String, ByteEntity>) createMock(RecordSupplier.class);
    rowIngestionMetersFactory = new TestUtils().getRowIngestionMetersFactory();
    supervisorConfig = new SupervisorStateManagerConfig();
    emitter = new TestEmitter();
    EasyMock.expect(spec.getSupervisorStateManagerConfig()).andReturn(supervisorConfig).anyTimes();
    EasyMock.expect(spec.getDataSchema()).andReturn(getDataSchema()).anyTimes();
    EasyMock.expect(spec.getIoConfig()).andReturn(getIOConfig()).anyTimes();
    EasyMock.expect(spec.getTuningConfig()).andReturn(getTuningConfig()).anyTimes();
    EasyMock.expect(spec.getEmitter()).andReturn(emitter).anyTimes();
    EasyMock.expect(taskClientFactory.build(EasyMock.anyObject(), EasyMock.anyString(), EasyMock.anyInt(), EasyMock.anyObject(), EasyMock.anyLong())).andReturn(indexTaskClient).anyTimes();
    EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.of(taskRunner)).anyTimes();
    EasyMock.expect(taskMaster.getTaskQueue()).andReturn(Optional.of(taskQueue)).anyTimes();
    taskRunner.registerListener(EasyMock.anyObject(TaskRunnerListener.class), EasyMock.anyObject(Executor.class));
    EasyMock.expect(indexerMetadataStorageCoordinator.retrieveDataSourceMetadata(DATASOURCE)).andReturn(null).anyTimes();
    EasyMock.expect(recordSupplier.getAssignment()).andReturn(ImmutableSet.of(SHARD0_PARTITION)).anyTimes();
    EasyMock.expect(recordSupplier.getLatestSequenceNumber(EasyMock.anyObject())).andReturn("10").anyTimes();
}
Also used : SeekableStreamIndexTaskClient(org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskClient) IndexerMetadataStorageCoordinator(org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator) TaskRunnerListener(org.apache.druid.indexing.overlord.TaskRunnerListener) ByteEntity(org.apache.druid.data.input.impl.ByteEntity) SeekableStreamIndexTaskClientFactory(org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskClientFactory) TaskRunner(org.apache.druid.indexing.overlord.TaskRunner) SeekableStreamIndexTaskRunner(org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskRunner) TestUtils(org.apache.druid.indexing.common.TestUtils) Executor(java.util.concurrent.Executor) TaskStorage(org.apache.druid.indexing.overlord.TaskStorage) SupervisorStateManagerConfig(org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig) TaskQueue(org.apache.druid.indexing.overlord.TaskQueue) TaskMaster(org.apache.druid.indexing.overlord.TaskMaster) Before(org.junit.Before)

Example 8 with TaskRunner

use of org.apache.druid.indexing.overlord.TaskRunner in project druid by druid-io.

the class OverlordResourceTest method setUp.

@Before
public void setUp() {
    taskRunner = EasyMock.createMock(TaskRunner.class);
    configManager = EasyMock.createMock(JacksonConfigManager.class);
    provisioningStrategy = EasyMock.createMock(ProvisioningStrategy.class);
    taskMaster = EasyMock.createStrictMock(TaskMaster.class);
    taskStorageQueryAdapter = EasyMock.createStrictMock(TaskStorageQueryAdapter.class);
    indexerMetadataStorageAdapter = EasyMock.createStrictMock(IndexerMetadataStorageAdapter.class);
    req = EasyMock.createStrictMock(HttpServletRequest.class);
    workerTaskRunnerQueryAdapter = EasyMock.createStrictMock(WorkerTaskRunnerQueryAdapter.class);
    EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.of(taskRunner)).anyTimes();
    AuthorizerMapper authMapper = new AuthorizerMapper(null) {

        @Override
        public Authorizer getAuthorizer(String name) {
            return new Authorizer() {

                @Override
                public Access authorize(AuthenticationResult authenticationResult, Resource resource, Action action) {
                    final String username = authenticationResult.getIdentity();
                    switch(resource.getName()) {
                        case "allow":
                            return new Access(true);
                        case Datasources.WIKIPEDIA:
                            // Only "Wiki Reader" can read "wikipedia"
                            return new Access(action == Action.READ && Users.WIKI_READER.equals(username));
                        case Datasources.BUZZFEED:
                            // Only "Buzz Reader" can read "buzzfeed"
                            return new Access(action == Action.READ && Users.BUZZ_READER.equals(username));
                        default:
                            return new Access(false);
                    }
                }
            };
        }
    };
    overlordResource = new OverlordResource(taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, null, configManager, null, authMapper, workerTaskRunnerQueryAdapter, provisioningStrategy);
}
Also used : IndexerMetadataStorageAdapter(org.apache.druid.indexing.overlord.IndexerMetadataStorageAdapter) Action(org.apache.druid.server.security.Action) JacksonConfigManager(org.apache.druid.common.config.JacksonConfigManager) Resource(org.apache.druid.server.security.Resource) Access(org.apache.druid.server.security.Access) TaskStorageQueryAdapter(org.apache.druid.indexing.overlord.TaskStorageQueryAdapter) TaskRunner(org.apache.druid.indexing.overlord.TaskRunner) WorkerTaskRunner(org.apache.druid.indexing.overlord.WorkerTaskRunner) ProvisioningStrategy(org.apache.druid.indexing.overlord.autoscaling.ProvisioningStrategy) AuthenticationResult(org.apache.druid.server.security.AuthenticationResult) HttpServletRequest(javax.servlet.http.HttpServletRequest) Authorizer(org.apache.druid.server.security.Authorizer) AuthorizerMapper(org.apache.druid.server.security.AuthorizerMapper) TaskMaster(org.apache.druid.indexing.overlord.TaskMaster) WorkerTaskRunnerQueryAdapter(org.apache.druid.indexing.overlord.WorkerTaskRunnerQueryAdapter) Before(org.junit.Before)

Aggregations

TaskRunner (org.apache.druid.indexing.overlord.TaskRunner)8 TaskMaster (org.apache.druid.indexing.overlord.TaskMaster)5 TaskQueue (org.apache.druid.indexing.overlord.TaskQueue)4 WorkerTaskRunner (org.apache.druid.indexing.overlord.WorkerTaskRunner)4 Before (org.junit.Before)4 ResourceFilters (com.sun.jersey.spi.container.ResourceFilters)3 GET (javax.ws.rs.GET)3 Path (javax.ws.rs.Path)3 Produces (javax.ws.rs.Produces)3 TaskStatus (org.apache.druid.indexer.TaskStatus)3 TestUtils (org.apache.druid.indexing.common.TestUtils)3 IndexerMetadataStorageCoordinator (org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator)3 TaskStorage (org.apache.druid.indexing.overlord.TaskStorage)3 SupervisorStateManagerConfig (org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig)3 SeekableStreamIndexTaskRunner (org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskRunner)3 HttpServletRequest (javax.servlet.http.HttpServletRequest)2 Response (javax.ws.rs.core.Response)2 Status (javax.ws.rs.core.Response.Status)2 JacksonConfigManager (org.apache.druid.common.config.JacksonConfigManager)2 TaskLocation (org.apache.druid.indexer.TaskLocation)2