Examples with JobContext org.apache.helix.task.JobContext used on opensource projects

Example 16 with JobContext

use of org.apache.helix.task.JobContext in project helix by apache.

the class TestRetrieveWorkflows method testGetAllWorkflows.

@Test
public void testGetAllWorkflows() throws Exception {
    List<Workflow> workflowList = new ArrayList<Workflow>();
    for (int i = 0; i < 2; i++) {
        Workflow workflow = WorkflowGenerator.generateDefaultRepeatedJobWorkflowBuilder(TestHelper.getTestMethodName() + i).build();
        _driver.start(workflow);
        workflowList.add(workflow);
    }
    for (Workflow workflow : workflowList) {
        _driver.pollForWorkflowState(workflow.getName(), TaskState.COMPLETED);
    }
    Map<String, WorkflowConfig> workflowConfigMap = _driver.getWorkflows();
    Assert.assertEquals(workflowConfigMap.size(), workflowList.size());
    for (Map.Entry<String, WorkflowConfig> workflow : workflowConfigMap.entrySet()) {
        WorkflowConfig workflowConfig = workflow.getValue();
        WorkflowContext workflowContext = _driver.getWorkflowContext(workflow.getKey());
        Assert.assertNotNull(workflowContext);
        for (String job : workflowConfig.getJobDag().getAllNodes()) {
            JobConfig jobConfig = _driver.getJobConfig(job);
            JobContext jobContext = _driver.getJobContext(job);
            Assert.assertNotNull(jobConfig);
            Assert.assertNotNull(jobContext);
        }
    }
}

Example 17 with JobContext

use of org.apache.helix.task.JobContext in project helix by apache.

the class TestTaskConditionalRetry method test.

@Test
public void test() throws Exception {
    int taskRetryCount = 5;
    int num_tasks = 5;
    String jobResource = TestHelper.getTestMethodName();
    JobConfig.Builder jobBuilder = new JobConfig.Builder();
    jobBuilder.setCommand(MockTask.TASK_COMMAND).setTimeoutPerTask(10000).setMaxAttemptsPerTask(taskRetryCount).setFailureThreshold(Integer.MAX_VALUE);
    // create each task configs.
    final int abortedTask = 1;
    final int failedTask = 2;
    final int exceptionTask = 3;
    List<TaskConfig> taskConfigs = new ArrayList<TaskConfig>();
    for (int j = 0; j < num_tasks; j++) {
        TaskConfig.Builder configBuilder = new TaskConfig.Builder().setTaskId("task_" + j);
        switch(j) {
            case abortedTask:
                configBuilder.addConfig(MockTask.TASK_RESULT_STATUS, TaskResult.Status.FATAL_FAILED.name());
                break;
            case failedTask:
                configBuilder.addConfig(MockTask.TASK_RESULT_STATUS, TaskResult.Status.FAILED.name());
                break;
            case exceptionTask:
                configBuilder.addConfig(MockTask.THROW_EXCEPTION, Boolean.TRUE.toString());
                break;
            default:
                break;
        }
        configBuilder.setTargetPartition(String.valueOf(j));
        taskConfigs.add(configBuilder.build());
    }
    jobBuilder.addTaskConfigs(taskConfigs);
    Workflow flow = WorkflowGenerator.generateSingleJobWorkflowBuilder(jobResource, jobBuilder).build();
    _driver.start(flow);
    // Wait until the job completes.
    _driver.pollForWorkflowState(jobResource, TaskState.COMPLETED);
    JobContext ctx = _driver.getJobContext(TaskUtil.getNamespacedJobName(jobResource));
    for (int i = 0; i < num_tasks; i++) {
        TaskPartitionState state = ctx.getPartitionState(i);
        int retriedCount = ctx.getPartitionNumAttempts(i);
        String taskId = ctx.getTaskIdForPartition(i);
        if (taskId.equals("task_" + abortedTask)) {
            Assert.assertEquals(state, TaskPartitionState.TASK_ABORTED);
            Assert.assertEquals(retriedCount, 1);
        } else if (taskId.equals("task_" + failedTask) || taskId.equals("task_" + exceptionTask)) {
            Assert.assertEquals(state, TaskPartitionState.TASK_ERROR);
            Assert.assertEquals(retriedCount, taskRetryCount);
        } else {
            Assert.assertEquals(state, TaskPartitionState.COMPLETED);
            Assert.assertEquals(retriedCount, 1);
        }
    }
}

Example 18 with JobContext

use of org.apache.helix.task.JobContext in project helix by apache.

the class TestTaskErrorReporting method test.

@Test
public void test() throws Exception {
    int taskRetryCount = 1;
    int num_tasks = 5;
    String jobResource = TestHelper.getTestMethodName();
    JobConfig.Builder jobBuilder = new JobConfig.Builder();
    jobBuilder.setCommand(MockTask.TASK_COMMAND).setTimeoutPerTask(10000).setMaxAttemptsPerTask(taskRetryCount).setFailureThreshold(Integer.MAX_VALUE);
    // create each task configs.
    final int abortedTask = 1;
    final int failedTask = 2;
    final int exceptionTask = 3;
    final String abortedMsg = "This task aborted, some terrible things must happened.";
    final String failedMsg = "This task failed, something may be wrong.";
    final String exceptionMsg = "This task throws exception.";
    final String successMsg = "Yes, we did it!";
    List<TaskConfig> taskConfigs = new ArrayList<TaskConfig>();
    for (int j = 0; j < num_tasks; j++) {
        TaskConfig.Builder configBuilder = new TaskConfig.Builder().setTaskId("task_" + j);
        switch(j) {
            case abortedTask:
                configBuilder.addConfig(MockTask.TASK_RESULT_STATUS, TaskResult.Status.FATAL_FAILED.name()).addConfig(MockTask.ERROR_MESSAGE, abortedMsg);
                break;
            case failedTask:
                configBuilder.addConfig(MockTask.TASK_RESULT_STATUS, TaskResult.Status.FAILED.name()).addConfig(MockTask.ERROR_MESSAGE, failedMsg);
                break;
            case exceptionTask:
                configBuilder.addConfig(MockTask.THROW_EXCEPTION, Boolean.TRUE.toString()).addConfig(MockTask.ERROR_MESSAGE, exceptionMsg);
                break;
            default:
                configBuilder.addConfig(MockTask.ERROR_MESSAGE, successMsg);
                break;
        }
        configBuilder.setTargetPartition(String.valueOf(j));
        taskConfigs.add(configBuilder.build());
    }
    jobBuilder.addTaskConfigs(taskConfigs);
    Workflow flow = WorkflowGenerator.generateSingleJobWorkflowBuilder(jobResource, jobBuilder).build();
    _driver.start(flow);
    // Wait until the job completes.
    _driver.pollForWorkflowState(jobResource, TaskState.COMPLETED);
    JobContext ctx = _driver.getJobContext(TaskUtil.getNamespacedJobName(jobResource));
    for (int i = 0; i < num_tasks; i++) {
        TaskPartitionState state = ctx.getPartitionState(i);
        String taskId = ctx.getTaskIdForPartition(i);
        String errMsg = ctx.getPartitionInfo(i);
        if (taskId.equals("task_" + abortedTask)) {
            Assert.assertEquals(state, TaskPartitionState.TASK_ABORTED);
            Assert.assertEquals(errMsg, abortedMsg);
        } else if (taskId.equals("task_" + failedTask)) {
            Assert.assertEquals(state, TaskPartitionState.TASK_ERROR);
            Assert.assertEquals(errMsg, failedMsg);
        } else if (taskId.equals("task_" + exceptionTask)) {
            Assert.assertEquals(state, TaskPartitionState.TASK_ERROR);
            Assert.assertTrue(errMsg.contains(exceptionMsg));
        } else {
            Assert.assertEquals(state, TaskPartitionState.COMPLETED);
            Assert.assertEquals(errMsg, successMsg);
        }
    }
}

Example 19 with JobContext

use of org.apache.helix.task.JobContext in project helix by apache.

the class TestIndependentTaskRebalancer method testDelayedRetry.

@Test
public void testDelayedRetry() throws Exception {
    // Create a single job with single task, set retry delay
    int delay = 3000;
    String jobName = TestHelper.getTestMethodName();
    Workflow.Builder workflowBuilder = new Workflow.Builder(jobName);
    List<TaskConfig> taskConfigs = Lists.newArrayListWithCapacity(1);
    Map<String, String> taskConfigMap = Maps.newHashMap();
    TaskConfig taskConfig1 = new TaskConfig("SingleFailTask", taskConfigMap);
    taskConfigs.add(taskConfig1);
    Map<String, String> jobCommandMap = Maps.newHashMap();
    JobConfig.Builder jobBuilder = new JobConfig.Builder().setCommand("DummyCommand").setTaskRetryDelay(delay).addTaskConfigs(taskConfigs).setJobCommandConfigMap(jobCommandMap);
    workflowBuilder.addJob(jobName, jobBuilder);
    SingleFailTask.hasFailed = false;
    _driver.start(workflowBuilder.build());
    // Ensure completion
    _driver.pollForWorkflowState(jobName, TaskState.COMPLETED);
    // Ensure a single retry happened
    JobContext jobCtx = _driver.getJobContext(jobName + "_" + jobName);
    Assert.assertEquals(jobCtx.getPartitionNumAttempts(0), 2);
    Assert.assertTrue(jobCtx.getFinishTime() - jobCtx.getStartTime() >= delay);
}

Example 20 with JobContext

use of org.apache.helix.task.JobContext in project helix by apache.

the class TestJobFailureHighThreshold method testHighThreshold.

/**
 * Number of instance is equal to number of failure threshold, thus job failure mechanism needs to consider given up
 * tasks that no longer exist on the instance, not only given up tasks currently reported on CurrentState.
 */
@Test
public void testHighThreshold() throws InterruptedException {
    final String WORKFLOW_NAME = "testWorkflow";
    final String JOB_NAME = "testJob";
    JobConfig.Builder jobBuilder = new JobConfig.Builder().setWorkflow(WORKFLOW_NAME).setTargetResource(DB_NAME).setTargetPartitionStates(Sets.newHashSet(MasterSlaveSMD.States.MASTER.name())).setCommand(MockTask.TASK_COMMAND).setJobCommandConfigMap(ImmutableMap.of(MockTask.TASK_RESULT_STATUS, TaskResult.Status.FATAL_FAILED.name())).setFailureThreshold(1);
    Workflow.Builder workflowBuilder = new Workflow.Builder(WORKFLOW_NAME).addJob(JOB_NAME, jobBuilder);
    _driver.start(workflowBuilder.build());
    _driver.pollForJobState(WORKFLOW_NAME, TaskUtil.getNamespacedJobName(WORKFLOW_NAME, JOB_NAME), TaskState.FAILED);
    _driver.pollForWorkflowState(WORKFLOW_NAME, TaskState.FAILED);
    JobContext jobContext = _driver.getJobContext(TaskUtil.getNamespacedJobName(WORKFLOW_NAME, JOB_NAME));
    int countAborted = 0;
    int countNoState = 0;
    for (int pId : jobContext.getPartitionSet()) {
        TaskPartitionState state = jobContext.getPartitionState(pId);
        if (state == TaskPartitionState.TASK_ABORTED) {
            countAborted++;
        } else if (state == null) {
            countNoState++;
        } else {
            throw new HelixException(String.format("State %s is not expected.", state));
        }
    }
    // Failure threshold is 1, so 2 tasks aborted.
    Assert.assertEquals(countAborted, 2);
    // Other 3 tasks are not scheduled at all.
    Assert.assertEquals(countNoState, 3);
}