Examples with TaskStatusPlus org.apache.druid.indexer.TaskStatusPlus used on opensource projects

Example 11 with TaskStatusPlus

use of org.apache.druid.indexer.TaskStatusPlus in project druid by druid-io.

the class ITS3OverrideCredentialsIndexTest method testS3WithoutOverrideCredentialsIndexDataShouldFailed.

@Test
public void testS3WithoutOverrideCredentialsIndexDataShouldFailed() throws Exception {
    final String indexDatasource = "wikipedia_index_test_" + UUID.randomUUID();
    try {
        final Function<String, String> s3PropsTransform = spec -> {
            try {
                String inputSourceValue = jsonMapper.writeValueAsString(INPUT_SOURCE_OBJECTS_VALUE);
                inputSourceValue = StringUtils.replace(inputSourceValue, "%%BUCKET%%", config.getCloudBucket());
                inputSourceValue = StringUtils.replace(inputSourceValue, "%%PATH%%", config.getCloudPath());
                spec = StringUtils.replace(spec, "%%INPUT_SOURCE_TYPE%%", "s3");
                spec = StringUtils.replace(spec, "%%INPUT_SOURCE_PROPERTY_KEY%%", INPUT_SOURCE_OBJECTS_KEY);
                return StringUtils.replace(spec, "%%INPUT_SOURCE_PROPERTY_VALUE%%", inputSourceValue);
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        };
        final String fullDatasourceName = indexDatasource + config.getExtraDatasourceNameSuffix();
        final String taskSpec = s3PropsTransform.apply(StringUtils.replace(getResourceAsString(INDEX_TASK_WITHOUT_OVERRIDE), "%%DATASOURCE%%", fullDatasourceName));
        final String taskID = indexer.submitTask(taskSpec);
        indexer.waitUntilTaskFails(taskID);
        TaskStatusPlus taskStatusPlus = indexer.getTaskStatus(taskID);
        // Index task is expected to fail as the default S3 Credentials in Druid's config (druid.s3.accessKey and
        // druid.s3.secretKey should not have access to the bucket and path for our data. (Refer to the setup instruction
        // at the top of this test class.
        Assert.assertEquals(taskStatusPlus.getStatusCode(), TaskState.FAILED);
        Assert.assertNotNull(taskStatusPlus.getErrorMsg());
        Assert.assertTrue(taskStatusPlus.getErrorMsg().contains("com.amazonaws.services.s3.model.AmazonS3Exception"), "Expect task to fail with AmazonS3Exception");
    } finally {
        // If the test pass, then there is no datasource to unload
        closeQuietly(unloader(indexDatasource + config.getExtraDatasourceNameSuffix()));
    }
}

Example 12 with TaskStatusPlus

use of org.apache.druid.indexer.TaskStatusPlus in project druid by druid-io.

the class ITS3OverrideCredentialsIndexTest method testS3WithInvalidOverrideCredentialsIndexDataShouldFailed.

@Test
public void testS3WithInvalidOverrideCredentialsIndexDataShouldFailed() throws Exception {
    final String indexDatasource = "wikipedia_index_test_" + UUID.randomUUID();
    try {
        final Function<String, String> s3PropsTransform = spec -> {
            try {
                String inputSourceValue = jsonMapper.writeValueAsString(INPUT_SOURCE_OBJECTS_VALUE);
                inputSourceValue = StringUtils.replace(inputSourceValue, "%%BUCKET%%", config.getCloudBucket());
                inputSourceValue = StringUtils.replace(inputSourceValue, "%%PATH%%", config.getCloudPath());
                spec = StringUtils.replace(spec, "%%INPUT_SOURCE_CONFIG%%", jsonMapper.writeValueAsString(ImmutableMap.of("accessKeyId", ImmutableMap.of("type", "environment", "variable", "INVALID_ACCESS_KEY"), "secretAccessKey", ImmutableMap.of("type", "environment", "variable", "INVALID_SECRET_KEY"))));
                spec = StringUtils.replace(spec, "%%INPUT_SOURCE_TYPE%%", "s3");
                spec = StringUtils.replace(spec, "%%INPUT_SOURCE_PROPERTY_KEY%%", INPUT_SOURCE_OBJECTS_KEY);
                return StringUtils.replace(spec, "%%INPUT_SOURCE_PROPERTY_VALUE%%", inputSourceValue);
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        };
        final String fullDatasourceName = indexDatasource + config.getExtraDatasourceNameSuffix();
        final String taskSpec = s3PropsTransform.apply(StringUtils.replace(getResourceAsString(INDEX_TASK_WITH_OVERRIDE), "%%DATASOURCE%%", fullDatasourceName));
        final String taskID = indexer.submitTask(taskSpec);
        indexer.waitUntilTaskFails(taskID);
        TaskStatusPlus taskStatusPlus = indexer.getTaskStatus(taskID);
        // Index task is expected to fail as the overrided s3 access key and s3 secret key cannot be null
        Assert.assertEquals(taskStatusPlus.getStatusCode(), TaskState.FAILED);
        Assert.assertNotNull(taskStatusPlus.getErrorMsg());
        Assert.assertTrue(taskStatusPlus.getErrorMsg().contains("IllegalArgumentException: Access key cannot be null"), "Expect task to fail with IllegalArgumentException: Access key cannot be null");
    } finally {
        // If the test pass, then there is no datasource to unload
        closeQuietly(unloader(indexDatasource + config.getExtraDatasourceNameSuffix()));
    }
}

Example 13 with TaskStatusPlus

use of org.apache.druid.indexer.TaskStatusPlus in project druid by druid-io.

the class OverlordTest method testOverlordRun.

@Test(timeout = 60_000L)
public void testOverlordRun() throws Exception {
    // basic task master lifecycle test
    taskMaster.start();
    announcementLatch.await();
    while (!taskMaster.isLeader()) {
        // I believe the control will never reach here and thread will never sleep but just to be on safe side
        Thread.sleep(10);
    }
    Assert.assertEquals(taskMaster.getCurrentLeader(), druidNode.getHostAndPort());
    final TaskStorageQueryAdapter taskStorageQueryAdapter = new TaskStorageQueryAdapter(taskStorage, taskLockbox);
    final WorkerTaskRunnerQueryAdapter workerTaskRunnerQueryAdapter = new WorkerTaskRunnerQueryAdapter(taskMaster, null);
    // Test Overlord resource stuff
    overlordResource = new OverlordResource(taskMaster, taskStorageQueryAdapter, new IndexerMetadataStorageAdapter(taskStorageQueryAdapter, null), null, null, null, AuthTestUtils.TEST_AUTHORIZER_MAPPER, workerTaskRunnerQueryAdapter, null);
    Response response = overlordResource.getLeader();
    Assert.assertEquals(druidNode.getHostAndPort(), response.getEntity());
    final String taskId_0 = "0";
    NoopTask task_0 = NoopTask.create(taskId_0, 0);
    response = overlordResource.taskPost(task_0, req);
    Assert.assertEquals(200, response.getStatus());
    Assert.assertEquals(ImmutableMap.of("task", taskId_0), response.getEntity());
    // Duplicate task - should fail
    response = overlordResource.taskPost(task_0, req);
    Assert.assertEquals(400, response.getStatus());
    // Task payload for task_0 should be present in taskStorage
    response = overlordResource.getTaskPayload(taskId_0);
    Assert.assertEquals(task_0, ((TaskPayloadResponse) response.getEntity()).getPayload());
    // Task not present in taskStorage - should fail
    response = overlordResource.getTaskPayload("whatever");
    Assert.assertEquals(404, response.getStatus());
    // Task status of the submitted task should be running
    response = overlordResource.getTaskStatus(taskId_0);
    Assert.assertEquals(taskId_0, ((TaskStatusResponse) response.getEntity()).getTask());
    Assert.assertEquals(TaskStatus.running(taskId_0).getStatusCode(), ((TaskStatusResponse) response.getEntity()).getStatus().getStatusCode());
    // Simulate completion of task_0
    taskCompletionCountDownLatches[Integer.parseInt(taskId_0)].countDown();
    // Wait for taskQueue to handle success status of task_0
    waitForTaskStatus(taskId_0, TaskState.SUCCESS);
    // Manually insert task in taskStorage
    // Verifies sync from storage
    final String taskId_1 = "1";
    NoopTask task_1 = NoopTask.create(taskId_1, 0);
    taskStorage.insert(task_1, TaskStatus.running(taskId_1));
    // Wait for task runner to run task_1
    runTaskCountDownLatches[Integer.parseInt(taskId_1)].await();
    response = overlordResource.getRunningTasks(null, req);
    // 1 task that was manually inserted should be in running state
    Assert.assertEquals(1, (((List) response.getEntity()).size()));
    final TaskStatusPlus taskResponseObject = ((List<TaskStatusPlus>) response.getEntity()).get(0);
    Assert.assertEquals(taskId_1, taskResponseObject.getId());
    Assert.assertEquals(TASK_LOCATION, taskResponseObject.getLocation());
    // Simulate completion of task_1
    taskCompletionCountDownLatches[Integer.parseInt(taskId_1)].countDown();
    // Wait for taskQueue to handle success status of task_1
    waitForTaskStatus(taskId_1, TaskState.SUCCESS);
    // should return number of tasks which are not in running state
    response = overlordResource.getCompleteTasks(null, req);
    Assert.assertEquals(2, (((List) response.getEntity()).size()));
    response = overlordResource.getCompleteTasks(1, req);
    Assert.assertEquals(1, (((List) response.getEntity()).size()));
    taskMaster.stop();
    Assert.assertFalse(taskMaster.isLeader());
    EasyMock.verify(taskLockbox, taskActionClientFactory);
}

Example 14 with TaskStatusPlus

use of org.apache.druid.indexer.TaskStatusPlus in project druid by druid-io.

the class OverlordResource method getTasks.

@GET
@Path("/tasks")
@Produces(MediaType.APPLICATION_JSON)
public Response getTasks(@QueryParam("state") final String state, @QueryParam("datasource") final String dataSource, @QueryParam("createdTimeInterval") final String createdTimeInterval, @QueryParam("max") final Integer maxCompletedTasks, @QueryParam("type") final String type, @Context final HttpServletRequest req) {
    // check for valid state
    if (state != null) {
        if (!API_TASK_STATES.contains(StringUtils.toLowerCase(state))) {
            return Response.status(Status.BAD_REQUEST).entity(StringUtils.format("Invalid state : %s, valid values are: %s", state, API_TASK_STATES)).build();
        }
    }
    // fail fast if user not authorized to access datasource
    if (dataSource != null) {
        final ResourceAction resourceAction = new ResourceAction(new Resource(dataSource, ResourceType.DATASOURCE), Action.READ);
        final Access authResult = AuthorizationUtils.authorizeResourceAction(req, resourceAction, authorizerMapper);
        if (!authResult.isAllowed()) {
            throw new WebApplicationException(Response.status(Response.Status.FORBIDDEN).entity(StringUtils.format("Access-Check-Result: %s", authResult.toString())).build());
        }
    }
    List<TaskStatusPlus> finalTaskList = new ArrayList<>();
    Function<AnyTask, TaskStatusPlus> activeTaskTransformFunc = workItem -> new TaskStatusPlus(workItem.getTaskId(), workItem.getTaskGroupId(), workItem.getTaskType(), workItem.getCreatedTime(), workItem.getQueueInsertionTime(), workItem.getTaskState(), workItem.getRunnerTaskState(), null, workItem.getLocation(), workItem.getDataSource(), null);
    Function<TaskInfo<Task, TaskStatus>, TaskStatusPlus> completeTaskTransformFunc = taskInfo -> new TaskStatusPlus(taskInfo.getId(), taskInfo.getTask() == null ? null : taskInfo.getTask().getGroupId(), taskInfo.getTask() == null ? null : taskInfo.getTask().getType(), taskInfo.getCreatedTime(), // TaskStorage API doesn't yet allow it.
    DateTimes.EPOCH, taskInfo.getStatus().getStatusCode(), RunnerTaskState.NONE, taskInfo.getStatus().getDuration(), taskInfo.getStatus().getLocation() == null ? TaskLocation.unknown() : taskInfo.getStatus().getLocation(), taskInfo.getDataSource(), taskInfo.getStatus().getErrorMsg());
    // checking for complete tasks first to avoid querying active tasks if user only wants complete tasks
    if (state == null || "complete".equals(StringUtils.toLowerCase(state))) {
        Duration createdTimeDuration = null;
        if (createdTimeInterval != null) {
            final Interval theInterval = Intervals.of(StringUtils.replace(createdTimeInterval, "_", "/"));
            createdTimeDuration = theInterval.toDuration();
        }
        final List<TaskInfo<Task, TaskStatus>> taskInfoList = taskStorageQueryAdapter.getCompletedTaskInfoByCreatedTimeDuration(maxCompletedTasks, createdTimeDuration, dataSource);
        final List<TaskStatusPlus> completedTasks = taskInfoList.stream().map(completeTaskTransformFunc::apply).collect(Collectors.toList());
        finalTaskList.addAll(completedTasks);
    }
    final List<TaskInfo<Task, TaskStatus>> allActiveTaskInfo;
    final List<AnyTask> allActiveTasks = new ArrayList<>();
    if (state == null || !"complete".equals(StringUtils.toLowerCase(state))) {
        allActiveTaskInfo = taskStorageQueryAdapter.getActiveTaskInfo(dataSource);
        for (final TaskInfo<Task, TaskStatus> task : allActiveTaskInfo) {
            allActiveTasks.add(new AnyTask(task.getId(), task.getTask() == null ? null : task.getTask().getGroupId(), task.getTask() == null ? null : task.getTask().getType(), SettableFuture.create(), task.getDataSource(), null, null, task.getCreatedTime(), DateTimes.EPOCH, TaskLocation.unknown()));
        }
    }
    if (state == null || "waiting".equals(StringUtils.toLowerCase(state))) {
        final List<AnyTask> waitingWorkItems = filterActiveTasks(RunnerTaskState.WAITING, allActiveTasks);
        List<TaskStatusPlus> transformedWaitingList = waitingWorkItems.stream().map(activeTaskTransformFunc::apply).collect(Collectors.toList());
        finalTaskList.addAll(transformedWaitingList);
    }
    if (state == null || "pending".equals(StringUtils.toLowerCase(state))) {
        final List<AnyTask> pendingWorkItems = filterActiveTasks(RunnerTaskState.PENDING, allActiveTasks);
        List<TaskStatusPlus> transformedPendingList = pendingWorkItems.stream().map(activeTaskTransformFunc::apply).collect(Collectors.toList());
        finalTaskList.addAll(transformedPendingList);
    }
    if (state == null || "running".equals(StringUtils.toLowerCase(state))) {
        final List<AnyTask> runningWorkItems = filterActiveTasks(RunnerTaskState.RUNNING, allActiveTasks);
        List<TaskStatusPlus> transformedRunningList = runningWorkItems.stream().map(activeTaskTransformFunc::apply).collect(Collectors.toList());
        finalTaskList.addAll(transformedRunningList);
    }
    final List<TaskStatusPlus> authorizedList = securedTaskStatusPlus(finalTaskList, dataSource, type, req);
    return Response.ok(authorizedList).build();
}

Example 15 with TaskStatusPlus

use of org.apache.druid.indexer.TaskStatusPlus in project druid by druid-io.

the class OverlordResource method getTaskStatus.

@GET
@Path("/task/{taskid}/status")
@Produces(MediaType.APPLICATION_JSON)
@ResourceFilters(TaskResourceFilter.class)
public Response getTaskStatus(@PathParam("taskid") String taskid) {
    final TaskInfo<Task, TaskStatus> taskInfo = taskStorageQueryAdapter.getTaskInfo(taskid);
    TaskStatusResponse response = null;
    if (taskInfo != null) {
        if (taskMaster.getTaskRunner().isPresent()) {
            final TaskRunner taskRunner = taskMaster.getTaskRunner().get();
            final TaskRunnerWorkItem workItem = taskRunner.getKnownTasks().stream().filter(item -> item.getTaskId().equals(taskid)).findAny().orElse(null);
            if (workItem != null) {
                response = new TaskStatusResponse(workItem.getTaskId(), new TaskStatusPlus(taskInfo.getId(), taskInfo.getTask() == null ? null : taskInfo.getTask().getGroupId(), taskInfo.getTask() == null ? null : taskInfo.getTask().getType(), taskInfo.getCreatedTime(), // TaskStorage API doesn't yet allow it.
                DateTimes.EPOCH, taskInfo.getStatus().getStatusCode(), taskRunner.getRunnerTaskState(workItem.getTaskId()), taskInfo.getStatus().getDuration(), workItem.getLocation(), taskInfo.getDataSource(), taskInfo.getStatus().getErrorMsg()));
            }
        }
        if (response == null) {
            response = new TaskStatusResponse(taskid, new TaskStatusPlus(taskInfo.getId(), taskInfo.getTask() == null ? null : taskInfo.getTask().getGroupId(), taskInfo.getTask() == null ? null : taskInfo.getTask().getType(), taskInfo.getCreatedTime(), // TaskStorage API doesn't yet allow it.
            DateTimes.EPOCH, taskInfo.getStatus().getStatusCode(), RunnerTaskState.WAITING, taskInfo.getStatus().getDuration(), taskInfo.getStatus().getLocation() == null ? TaskLocation.unknown() : taskInfo.getStatus().getLocation(), taskInfo.getDataSource(), taskInfo.getStatus().getErrorMsg()));
        }
    } else {
        response = new TaskStatusResponse(taskid, null);
    }
    final Response.Status status = response.getStatus() == null ? Response.Status.NOT_FOUND : Response.Status.OK;
    return Response.status(status).entity(response).build();
}