Examples with TaskPartitionState org.apache.helix.task.TaskPartitionState used on opensource projects

Example 6 with TaskPartitionState

use of org.apache.helix.task.TaskPartitionState in project helix by apache.

the class TaskTestUtil method pollForWorkflowParallelState.

// 1. Different jobs in a same work flow is in RUNNING at the same time
// 2. When disallow overlap assignment, no two jobs in the same work flow is in RUNNING at the same instance
// Use this method with caution because it assumes workflow doesn't finish too quickly and number of parallel running
// tasks can be counted.
public static boolean pollForWorkflowParallelState(TaskDriver driver, String workflowName) throws InterruptedException {
    WorkflowConfig workflowConfig = driver.getWorkflowConfig(workflowName);
    Assert.assertNotNull(workflowConfig);
    WorkflowContext workflowContext = null;
    while (workflowContext == null) {
        workflowContext = driver.getWorkflowContext(workflowName);
        Thread.sleep(100);
    }
    int maxRunningCount = 0;
    boolean finished = false;
    while (!finished) {
        finished = true;
        int runningCount = 0;
        workflowContext = driver.getWorkflowContext(workflowName);
        for (String jobName : workflowConfig.getJobDag().getAllNodes()) {
            TaskState jobState = workflowContext.getJobState(jobName);
            if (jobState == TaskState.IN_PROGRESS) {
                ++runningCount;
                finished = false;
            }
        }
        if (runningCount > maxRunningCount) {
            maxRunningCount = runningCount;
        }
        List<JobContext> jobContextList = new ArrayList<JobContext>();
        for (String jobName : workflowConfig.getJobDag().getAllNodes()) {
            JobContext jobContext = driver.getJobContext(jobName);
            if (jobContext != null) {
                jobContextList.add(driver.getJobContext(jobName));
            }
        }
        if (!workflowConfig.isAllowOverlapJobAssignment()) {
            Set<String> instances = new HashSet<String>();
            for (JobContext jobContext : jobContextList) {
                for (int partition : jobContext.getPartitionSet()) {
                    String instance = jobContext.getAssignedParticipant(partition);
                    TaskPartitionState taskPartitionState = jobContext.getPartitionState(partition);
                    if (instance == null) {
                        continue;
                    }
                    if (taskPartitionState != TaskPartitionState.INIT && taskPartitionState != TaskPartitionState.RUNNING) {
                        continue;
                    }
                    if (instances.contains(instance)) {
                        return false;
                    }
                    TaskPartitionState state = jobContext.getPartitionState(partition);
                    if (state != TaskPartitionState.COMPLETED) {
                        instances.add(instance);
                    }
                }
            }
        }
        Thread.sleep(100);
    }
    return maxRunningCount > 1 && (workflowConfig.isJobQueue() ? maxRunningCount <= workflowConfig.getParallelJobs() : true);
}

Example 7 with TaskPartitionState

use of org.apache.helix.task.TaskPartitionState in project helix by apache.

the class TestTaskRebalancer method timeouts.

@Test
public void timeouts() throws Exception {
    final String jobResource = "timeouts";
    JobConfig.Builder jobBuilder = JobConfig.Builder.fromMap(WorkflowGenerator.DEFAULT_JOB_CONFIG);
    jobBuilder.setJobCommandConfigMap(WorkflowGenerator.DEFAULT_COMMAND_CONFIG).setMaxAttemptsPerTask(2).setTimeoutPerTask(100);
    Workflow flow = WorkflowGenerator.generateSingleJobWorkflowBuilder(jobResource, jobBuilder).build();
    _driver.start(flow);
    // Wait until the job reports failure.
    _driver.pollForWorkflowState(jobResource, TaskState.FAILED);
    // Check that all partitions timed out up to maxAttempts
    JobContext ctx = _driver.getJobContext(TaskUtil.getNamespacedJobName(jobResource));
    int maxAttempts = 0;
    boolean sawTimedoutTask = false;
    for (int i = 0; i < _numParitions; i++) {
        TaskPartitionState state = ctx.getPartitionState(i);
        if (state != null) {
            if (state == TaskPartitionState.TIMED_OUT) {
                sawTimedoutTask = true;
            }
            // At least one task timed out, other might be aborted due to job failure.
            Assert.assertTrue(state == TaskPartitionState.TIMED_OUT || state == TaskPartitionState.TASK_ABORTED);
            maxAttempts = Math.max(maxAttempts, ctx.getPartitionNumAttempts(i));
        }
    }
    Assert.assertTrue(sawTimedoutTask);
    Assert.assertEquals(maxAttempts, 2);
}

Example 8 with TaskPartitionState

use of org.apache.helix.task.TaskPartitionState in project helix by apache.

the class TestTaskRebalancerRetryLimit method test.

@Test
public void test() throws Exception {
    String jobResource = TestHelper.getTestMethodName();
    JobConfig.Builder jobBuilder = JobConfig.Builder.fromMap(WorkflowGenerator.DEFAULT_JOB_CONFIG);
    jobBuilder.setJobCommandConfigMap(WorkflowGenerator.DEFAULT_COMMAND_CONFIG).setMaxAttemptsPerTask(2).setCommand(MockTask.TASK_COMMAND).setFailureThreshold(Integer.MAX_VALUE).setJobCommandConfigMap(ImmutableMap.of(MockTask.THROW_EXCEPTION, "true"));
    Workflow flow = WorkflowGenerator.generateSingleJobWorkflowBuilder(jobResource, jobBuilder).build();
    _driver.start(flow);
    // Wait until the job completes.
    _driver.pollForWorkflowState(jobResource, TaskState.COMPLETED);
    JobContext ctx = _driver.getJobContext(TaskUtil.getNamespacedJobName(jobResource));
    for (int i = 0; i < _numParitions; i++) {
        TaskPartitionState state = ctx.getPartitionState(i);
        if (state != null) {
            Assert.assertEquals(state, TaskPartitionState.TASK_ERROR);
            Assert.assertEquals(ctx.getPartitionNumAttempts(i), 2);
        }
    }
}