Examples with TaskInfo org.apache.druid.indexer.TaskInfo used on opensource projects

Example 1 with TaskInfo

use of org.apache.druid.indexer.TaskInfo in project druid by druid-io.

the class HeapMemoryTaskStorage method getActiveTaskInfo.

@Override
public List<TaskInfo<Task, TaskStatus>> getActiveTaskInfo(@Nullable String dataSource) {
    final ImmutableList.Builder<TaskInfo<Task, TaskStatus>> listBuilder = ImmutableList.builder();
    for (final TaskStuff taskStuff : tasks.values()) {
        if (taskStuff.getStatus().isRunnable()) {
            TaskInfo t = TaskStuff.toTaskInfo(taskStuff);
            listBuilder.add(t);
        }
    }
    return listBuilder.build();
}

Example 2 with TaskInfo

use of org.apache.druid.indexer.TaskInfo in project druid by druid-io.

the class IndexerMetadataStorageAdapterTest method testDeletePendingSegments.

@Test
public void testDeletePendingSegments() {
    final List<TaskInfo<Task, TaskStatus>> taskInfos = ImmutableList.of(new TaskInfo<>("id1", DateTimes.of("2017-12-01"), TaskStatus.running("id1"), "dataSource", NoopTask.create("id1", 0)), new TaskInfo<>("id1", DateTimes.of("2017-12-02"), TaskStatus.running("id2"), "dataSource", NoopTask.create("id2", 0)));
    EasyMock.expect(taskStorageQueryAdapter.getActiveTaskInfo("dataSource")).andReturn(taskInfos);
    final Interval deleteInterval = Intervals.of("2017-01-01/2017-12-01");
    EasyMock.expect(indexerMetadataStorageCoordinator.deletePendingSegmentsCreatedInInterval(EasyMock.anyString(), EasyMock.eq(deleteInterval))).andReturn(10);
    EasyMock.replay(taskStorageQueryAdapter, indexerMetadataStorageCoordinator);
    Assert.assertEquals(10, indexerMetadataStorageAdapter.deletePendingSegments("dataSource", deleteInterval));
}

Example 3 with TaskInfo

use of org.apache.druid.indexer.TaskInfo in project druid by druid-io.

the class IndexerMetadataStorageAdapterTest method testDeletePendingSegmentsOfRunningTasks.

@Test
public void testDeletePendingSegmentsOfRunningTasks() {
    final ImmutableList<TaskInfo<Task, TaskStatus>> taskInfos = ImmutableList.of(new TaskInfo<>("id1", DateTimes.of("2017-11-01"), TaskStatus.running("id1"), "dataSource", NoopTask.create("id1", 0)), new TaskInfo<>("id1", DateTimes.of("2017-12-02"), TaskStatus.running("id2"), "dataSource", NoopTask.create("id2", 0)));
    EasyMock.expect(taskStorageQueryAdapter.getActiveTaskInfo("dataSource")).andReturn(taskInfos);
    final Interval deleteInterval = Intervals.of("2017-01-01/2017-12-01");
    EasyMock.expect(indexerMetadataStorageCoordinator.deletePendingSegmentsCreatedInInterval(EasyMock.anyString(), EasyMock.eq(deleteInterval))).andReturn(10);
    EasyMock.replay(taskStorageQueryAdapter, indexerMetadataStorageCoordinator);
    expectedException.expect(CoreMatchers.instanceOf(IllegalArgumentException.class));
    expectedException.expectMessage("Cannot delete pendingSegments because there is at least one active task created");
    indexerMetadataStorageAdapter.deletePendingSegments("dataSource", deleteInterval);
}

Example 4 with TaskInfo

use of org.apache.druid.indexer.TaskInfo in project druid by druid-io.

the class OverlordResource method getTasks.

@GET
@Path("/tasks")
@Produces(MediaType.APPLICATION_JSON)
public Response getTasks(@QueryParam("state") final String state, @QueryParam("datasource") final String dataSource, @QueryParam("createdTimeInterval") final String createdTimeInterval, @QueryParam("max") final Integer maxCompletedTasks, @QueryParam("type") final String type, @Context final HttpServletRequest req) {
    // check for valid state
    if (state != null) {
        if (!API_TASK_STATES.contains(StringUtils.toLowerCase(state))) {
            return Response.status(Status.BAD_REQUEST).entity(StringUtils.format("Invalid state : %s, valid values are: %s", state, API_TASK_STATES)).build();
        }
    }
    // fail fast if user not authorized to access datasource
    if (dataSource != null) {
        final ResourceAction resourceAction = new ResourceAction(new Resource(dataSource, ResourceType.DATASOURCE), Action.READ);
        final Access authResult = AuthorizationUtils.authorizeResourceAction(req, resourceAction, authorizerMapper);
        if (!authResult.isAllowed()) {
            throw new WebApplicationException(Response.status(Response.Status.FORBIDDEN).entity(StringUtils.format("Access-Check-Result: %s", authResult.toString())).build());
        }
    }
    List<TaskStatusPlus> finalTaskList = new ArrayList<>();
    Function<AnyTask, TaskStatusPlus> activeTaskTransformFunc = workItem -> new TaskStatusPlus(workItem.getTaskId(), workItem.getTaskGroupId(), workItem.getTaskType(), workItem.getCreatedTime(), workItem.getQueueInsertionTime(), workItem.getTaskState(), workItem.getRunnerTaskState(), null, workItem.getLocation(), workItem.getDataSource(), null);
    Function<TaskInfo<Task, TaskStatus>, TaskStatusPlus> completeTaskTransformFunc = taskInfo -> new TaskStatusPlus(taskInfo.getId(), taskInfo.getTask() == null ? null : taskInfo.getTask().getGroupId(), taskInfo.getTask() == null ? null : taskInfo.getTask().getType(), taskInfo.getCreatedTime(), // TaskStorage API doesn't yet allow it.
    DateTimes.EPOCH, taskInfo.getStatus().getStatusCode(), RunnerTaskState.NONE, taskInfo.getStatus().getDuration(), taskInfo.getStatus().getLocation() == null ? TaskLocation.unknown() : taskInfo.getStatus().getLocation(), taskInfo.getDataSource(), taskInfo.getStatus().getErrorMsg());
    // checking for complete tasks first to avoid querying active tasks if user only wants complete tasks
    if (state == null || "complete".equals(StringUtils.toLowerCase(state))) {
        Duration createdTimeDuration = null;
        if (createdTimeInterval != null) {
            final Interval theInterval = Intervals.of(StringUtils.replace(createdTimeInterval, "_", "/"));
            createdTimeDuration = theInterval.toDuration();
        }
        final List<TaskInfo<Task, TaskStatus>> taskInfoList = taskStorageQueryAdapter.getCompletedTaskInfoByCreatedTimeDuration(maxCompletedTasks, createdTimeDuration, dataSource);
        final List<TaskStatusPlus> completedTasks = taskInfoList.stream().map(completeTaskTransformFunc::apply).collect(Collectors.toList());
        finalTaskList.addAll(completedTasks);
    }
    final List<TaskInfo<Task, TaskStatus>> allActiveTaskInfo;
    final List<AnyTask> allActiveTasks = new ArrayList<>();
    if (state == null || !"complete".equals(StringUtils.toLowerCase(state))) {
        allActiveTaskInfo = taskStorageQueryAdapter.getActiveTaskInfo(dataSource);
        for (final TaskInfo<Task, TaskStatus> task : allActiveTaskInfo) {
            allActiveTasks.add(new AnyTask(task.getId(), task.getTask() == null ? null : task.getTask().getGroupId(), task.getTask() == null ? null : task.getTask().getType(), SettableFuture.create(), task.getDataSource(), null, null, task.getCreatedTime(), DateTimes.EPOCH, TaskLocation.unknown()));
        }
    }
    if (state == null || "waiting".equals(StringUtils.toLowerCase(state))) {
        final List<AnyTask> waitingWorkItems = filterActiveTasks(RunnerTaskState.WAITING, allActiveTasks);
        List<TaskStatusPlus> transformedWaitingList = waitingWorkItems.stream().map(activeTaskTransformFunc::apply).collect(Collectors.toList());
        finalTaskList.addAll(transformedWaitingList);
    }
    if (state == null || "pending".equals(StringUtils.toLowerCase(state))) {
        final List<AnyTask> pendingWorkItems = filterActiveTasks(RunnerTaskState.PENDING, allActiveTasks);
        List<TaskStatusPlus> transformedPendingList = pendingWorkItems.stream().map(activeTaskTransformFunc::apply).collect(Collectors.toList());
        finalTaskList.addAll(transformedPendingList);
    }
    if (state == null || "running".equals(StringUtils.toLowerCase(state))) {
        final List<AnyTask> runningWorkItems = filterActiveTasks(RunnerTaskState.RUNNING, allActiveTasks);
        List<TaskStatusPlus> transformedRunningList = runningWorkItems.stream().map(activeTaskTransformFunc::apply).collect(Collectors.toList());
        finalTaskList.addAll(transformedRunningList);
    }
    final List<TaskStatusPlus> authorizedList = securedTaskStatusPlus(finalTaskList, dataSource, type, req);
    return Response.ok(authorizedList).build();
}

Example 5 with TaskInfo

use of org.apache.druid.indexer.TaskInfo in project druid by druid-io.

the class OverlordResourceTest method testGetTaskStatus.

@Test
public void testGetTaskStatus() throws Exception {
    // This is disabled since OverlordResource.getTaskStatus() is annotated with TaskResourceFilter which is supposed to
    // set authorization token properly, but isn't called in this test.
    // This should be fixed in https://github.com/apache/druid/issues/6685.
    // expectAuthorizationTokenCheck();
    final Task task = NoopTask.create("mytask", 0);
    final TaskStatus status = TaskStatus.running("mytask");
    EasyMock.expect(taskStorageQueryAdapter.getTaskInfo("mytask")).andReturn(new TaskInfo(task.getId(), DateTimes.of("2018-01-01"), status, task.getDataSource(), task));
    EasyMock.expect(taskStorageQueryAdapter.getTaskInfo("othertask")).andReturn(null);
    EasyMock.<Collection<? extends TaskRunnerWorkItem>>expect(taskRunner.getKnownTasks()).andReturn(ImmutableList.of());
    EasyMock.replay(taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req, workerTaskRunnerQueryAdapter);
    final Response response1 = overlordResource.getTaskStatus("mytask");
    final TaskStatusResponse taskStatusResponse1 = TestHelper.makeJsonMapper().readValue(TestHelper.makeJsonMapper().writeValueAsString(response1.getEntity()), TaskStatusResponse.class);
    TaskStatusPlus tsp = taskStatusResponse1.getStatus();
    Assert.assertEquals(tsp.getStatusCode(), tsp.getStatus());
    Assert.assertEquals(new TaskStatusResponse("mytask", new TaskStatusPlus("mytask", "mytask", "noop", DateTimes.of("2018-01-01"), DateTimes.EPOCH, TaskState.RUNNING, RunnerTaskState.RUNNING, -1L, TaskLocation.unknown(), task.getDataSource(), null)), taskStatusResponse1);
    final Response response2 = overlordResource.getTaskStatus("othertask");
    final TaskStatusResponse taskStatusResponse2 = TestHelper.makeJsonMapper().readValue(TestHelper.makeJsonMapper().writeValueAsString(response2.getEntity()), TaskStatusResponse.class);
    Assert.assertEquals(new TaskStatusResponse("othertask", null), taskStatusResponse2);
}