Examples with TaskPartitionState org.apache.helix.task.TaskPartitionState used on opensource projects

Example 1 with TaskPartitionState

use of org.apache.helix.task.TaskPartitionState in project helix by apache.

the class TaskAdmin method list.

private static void list(TaskDriver taskDriver, String workflow) {
    WorkflowConfig wCfg = taskDriver.getWorkflowConfig(workflow);
    if (wCfg == null) {
        LOG.error("Workflow " + workflow + " does not exist!");
        return;
    }
    WorkflowContext wCtx = taskDriver.getWorkflowContext(workflow);
    LOG.info("Workflow " + workflow + " consists of the following tasks: " + wCfg.getJobDag().getAllNodes());
    String workflowState = (wCtx != null) ? wCtx.getWorkflowState().name() : TaskState.NOT_STARTED.name();
    LOG.info("Current state of workflow is " + workflowState);
    LOG.info("Job states are: ");
    LOG.info("-------");
    for (String job : wCfg.getJobDag().getAllNodes()) {
        TaskState jobState = (wCtx != null) ? wCtx.getJobState(job) : TaskState.NOT_STARTED;
        LOG.info("Job " + job + " is " + jobState);
        // fetch job information
        JobConfig jCfg = taskDriver.getJobConfig(job);
        JobContext jCtx = taskDriver.getJobContext(job);
        if (jCfg == null || jCtx == null) {
            LOG.info("-------");
            continue;
        }
        // calculate taskPartitions
        List<Integer> partitions = Lists.newArrayList(jCtx.getPartitionSet());
        Collections.sort(partitions);
        // report status
        for (Integer partition : partitions) {
            String taskId = jCtx.getTaskIdForPartition(partition);
            taskId = (taskId != null) ? taskId : jCtx.getTargetForPartition(partition);
            LOG.info("Task: " + taskId);
            TaskConfig taskConfig = jCfg.getTaskConfig(taskId);
            if (taskConfig != null) {
                LOG.info("Configuration: " + taskConfig.getConfigMap());
            }
            TaskPartitionState state = jCtx.getPartitionState(partition);
            state = (state != null) ? state : TaskPartitionState.INIT;
            LOG.info("State: " + state);
            String assignedParticipant = jCtx.getAssignedParticipant(partition);
            if (assignedParticipant != null) {
                LOG.info("Assigned participant: " + assignedParticipant);
            }
            LOG.info("-------");
        }
        LOG.info("-------");
    }
}

Example 2 with TaskPartitionState

use of org.apache.helix.task.TaskPartitionState in project helix by apache.

the class TaskTestUtil method buildJobContext.

public static JobContext buildJobContext(Long startTime, Long finishTime, TaskPartitionState... partitionStates) {
    JobContext jobContext = new JobContext(new ZNRecord(TaskUtil.TASK_CONTEXT_KW));
    jobContext.setStartTime(startTime == null ? System.currentTimeMillis() : startTime);
    jobContext.setFinishTime(finishTime == null ? System.currentTimeMillis() : finishTime);
    int partitionId = 0;
    for (TaskPartitionState partitionState : partitionStates) {
        jobContext.setPartitionState(partitionId++, partitionState);
    }
    return jobContext;
}

Example 3 with TaskPartitionState

use of org.apache.helix.task.TaskPartitionState in project helix by apache.

the class TestTaskConditionalRetry method test.

@Test
public void test() throws Exception {
    int taskRetryCount = 5;
    int num_tasks = 5;
    String jobResource = TestHelper.getTestMethodName();
    JobConfig.Builder jobBuilder = new JobConfig.Builder();
    jobBuilder.setCommand(MockTask.TASK_COMMAND).setTimeoutPerTask(10000).setMaxAttemptsPerTask(taskRetryCount).setFailureThreshold(Integer.MAX_VALUE);
    // create each task configs.
    final int abortedTask = 1;
    final int failedTask = 2;
    final int exceptionTask = 3;
    List<TaskConfig> taskConfigs = new ArrayList<TaskConfig>();
    for (int j = 0; j < num_tasks; j++) {
        TaskConfig.Builder configBuilder = new TaskConfig.Builder().setTaskId("task_" + j);
        switch(j) {
            case abortedTask:
                configBuilder.addConfig(MockTask.TASK_RESULT_STATUS, TaskResult.Status.FATAL_FAILED.name());
                break;
            case failedTask:
                configBuilder.addConfig(MockTask.TASK_RESULT_STATUS, TaskResult.Status.FAILED.name());
                break;
            case exceptionTask:
                configBuilder.addConfig(MockTask.THROW_EXCEPTION, Boolean.TRUE.toString());
                break;
            default:
                break;
        }
        configBuilder.setTargetPartition(String.valueOf(j));
        taskConfigs.add(configBuilder.build());
    }
    jobBuilder.addTaskConfigs(taskConfigs);
    Workflow flow = WorkflowGenerator.generateSingleJobWorkflowBuilder(jobResource, jobBuilder).build();
    _driver.start(flow);
    // Wait until the job completes.
    _driver.pollForWorkflowState(jobResource, TaskState.COMPLETED);
    JobContext ctx = _driver.getJobContext(TaskUtil.getNamespacedJobName(jobResource));
    for (int i = 0; i < num_tasks; i++) {
        TaskPartitionState state = ctx.getPartitionState(i);
        int retriedCount = ctx.getPartitionNumAttempts(i);
        String taskId = ctx.getTaskIdForPartition(i);
        if (taskId.equals("task_" + abortedTask)) {
            Assert.assertEquals(state, TaskPartitionState.TASK_ABORTED);
            Assert.assertEquals(retriedCount, 1);
        } else if (taskId.equals("task_" + failedTask) || taskId.equals("task_" + exceptionTask)) {
            Assert.assertEquals(state, TaskPartitionState.TASK_ERROR);
            Assert.assertEquals(retriedCount, taskRetryCount);
        } else {
            Assert.assertEquals(state, TaskPartitionState.COMPLETED);
            Assert.assertEquals(retriedCount, 1);
        }
    }
}

Example 4 with TaskPartitionState

use of org.apache.helix.task.TaskPartitionState in project helix by apache.

the class TestTaskErrorReporting method test.

@Test
public void test() throws Exception {
    int taskRetryCount = 1;
    int num_tasks = 5;
    String jobResource = TestHelper.getTestMethodName();
    JobConfig.Builder jobBuilder = new JobConfig.Builder();
    jobBuilder.setCommand(MockTask.TASK_COMMAND).setTimeoutPerTask(10000).setMaxAttemptsPerTask(taskRetryCount).setFailureThreshold(Integer.MAX_VALUE);
    // create each task configs.
    final int abortedTask = 1;
    final int failedTask = 2;
    final int exceptionTask = 3;
    final String abortedMsg = "This task aborted, some terrible things must happened.";
    final String failedMsg = "This task failed, something may be wrong.";
    final String exceptionMsg = "This task throws exception.";
    final String successMsg = "Yes, we did it!";
    List<TaskConfig> taskConfigs = new ArrayList<TaskConfig>();
    for (int j = 0; j < num_tasks; j++) {
        TaskConfig.Builder configBuilder = new TaskConfig.Builder().setTaskId("task_" + j);
        switch(j) {
            case abortedTask:
                configBuilder.addConfig(MockTask.TASK_RESULT_STATUS, TaskResult.Status.FATAL_FAILED.name()).addConfig(MockTask.ERROR_MESSAGE, abortedMsg);
                break;
            case failedTask:
                configBuilder.addConfig(MockTask.TASK_RESULT_STATUS, TaskResult.Status.FAILED.name()).addConfig(MockTask.ERROR_MESSAGE, failedMsg);
                break;
            case exceptionTask:
                configBuilder.addConfig(MockTask.THROW_EXCEPTION, Boolean.TRUE.toString()).addConfig(MockTask.ERROR_MESSAGE, exceptionMsg);
                break;
            default:
                configBuilder.addConfig(MockTask.ERROR_MESSAGE, successMsg);
                break;
        }
        configBuilder.setTargetPartition(String.valueOf(j));
        taskConfigs.add(configBuilder.build());
    }
    jobBuilder.addTaskConfigs(taskConfigs);
    Workflow flow = WorkflowGenerator.generateSingleJobWorkflowBuilder(jobResource, jobBuilder).build();
    _driver.start(flow);
    // Wait until the job completes.
    _driver.pollForWorkflowState(jobResource, TaskState.COMPLETED);
    JobContext ctx = _driver.getJobContext(TaskUtil.getNamespacedJobName(jobResource));
    for (int i = 0; i < num_tasks; i++) {
        TaskPartitionState state = ctx.getPartitionState(i);
        String taskId = ctx.getTaskIdForPartition(i);
        String errMsg = ctx.getPartitionInfo(i);
        if (taskId.equals("task_" + abortedTask)) {
            Assert.assertEquals(state, TaskPartitionState.TASK_ABORTED);
            Assert.assertEquals(errMsg, abortedMsg);
        } else if (taskId.equals("task_" + failedTask)) {
            Assert.assertEquals(state, TaskPartitionState.TASK_ERROR);
            Assert.assertEquals(errMsg, failedMsg);
        } else if (taskId.equals("task_" + exceptionTask)) {
            Assert.assertEquals(state, TaskPartitionState.TASK_ERROR);
            Assert.assertTrue(errMsg.contains(exceptionMsg));
        } else {
            Assert.assertEquals(state, TaskPartitionState.COMPLETED);
            Assert.assertEquals(errMsg, successMsg);
        }
    }
}

Example 5 with TaskPartitionState

use of org.apache.helix.task.TaskPartitionState in project helix by apache.

the class TestJobFailureHighThreshold method testHighThreshold.

/**
 * Number of instance is equal to number of failure threshold, thus job failure mechanism needs to consider given up
 * tasks that no longer exist on the instance, not only given up tasks currently reported on CurrentState.
 */
@Test
public void testHighThreshold() throws InterruptedException {
    final String WORKFLOW_NAME = "testWorkflow";
    final String JOB_NAME = "testJob";
    JobConfig.Builder jobBuilder = new JobConfig.Builder().setWorkflow(WORKFLOW_NAME).setTargetResource(DB_NAME).setTargetPartitionStates(Sets.newHashSet(MasterSlaveSMD.States.MASTER.name())).setCommand(MockTask.TASK_COMMAND).setJobCommandConfigMap(ImmutableMap.of(MockTask.TASK_RESULT_STATUS, TaskResult.Status.FATAL_FAILED.name())).setFailureThreshold(1);
    Workflow.Builder workflowBuilder = new Workflow.Builder(WORKFLOW_NAME).addJob(JOB_NAME, jobBuilder);
    _driver.start(workflowBuilder.build());
    _driver.pollForJobState(WORKFLOW_NAME, TaskUtil.getNamespacedJobName(WORKFLOW_NAME, JOB_NAME), TaskState.FAILED);
    _driver.pollForWorkflowState(WORKFLOW_NAME, TaskState.FAILED);
    JobContext jobContext = _driver.getJobContext(TaskUtil.getNamespacedJobName(WORKFLOW_NAME, JOB_NAME));
    int countAborted = 0;
    int countNoState = 0;
    for (int pId : jobContext.getPartitionSet()) {
        TaskPartitionState state = jobContext.getPartitionState(pId);
        if (state == TaskPartitionState.TASK_ABORTED) {
            countAborted++;
        } else if (state == null) {
            countNoState++;
        } else {
            throw new HelixException(String.format("State %s is not expected.", state));
        }
    }
    // Failure threshold is 1, so 2 tasks aborted.
    Assert.assertEquals(countAborted, 2);
    // Other 3 tasks are not scheduled at all.
    Assert.assertEquals(countNoState, 3);
}