Examples with Workflow org.apache.helix.task.Workflow used on opensource projects

Example 6 with Workflow

use of org.apache.helix.task.Workflow in project helix by apache.

the class TestTaskRetryDelay method testTaskRetryWithoutDelay.

@Test
public void testTaskRetryWithoutDelay() throws Exception {
    String jobResource = TestHelper.getTestMethodName();
    JobConfig.Builder jobBuilder = JobConfig.Builder.fromMap(WorkflowGenerator.DEFAULT_JOB_CONFIG);
    jobBuilder.setJobCommandConfigMap(WorkflowGenerator.DEFAULT_COMMAND_CONFIG).setMaxAttemptsPerTask(2).setCommand(MockTask.TASK_COMMAND).setFailureThreshold(Integer.MAX_VALUE).setJobCommandConfigMap(ImmutableMap.of(MockTask.FAILURE_COUNT_BEFORE_SUCCESS, "1"));
    Workflow flow = WorkflowGenerator.generateSingleJobWorkflowBuilder(jobResource, jobBuilder).build();
    _driver.start(flow);
    // Wait until the job completes.
    _driver.pollForWorkflowState(jobResource, TaskState.COMPLETED);
    long startTime = _driver.getWorkflowContext(jobResource).getStartTime();
    long finishedTime = _driver.getWorkflowContext(jobResource).getFinishTime();
    // It should finished at less than 2 sec
    Assert.assertTrue(finishedTime - startTime <= 2000L);
}

Example 7 with Workflow

use of org.apache.helix.task.Workflow in project helix by apache.

the class TestTaskThrottling method testJobPriority.

@Test(dependsOnMethods = { "testTaskThrottle" })
public void testJobPriority() throws InterruptedException {
    int numTasks = 30 * _numNodes;
    int perNodeTaskLimitation = 5;
    JobConfig.Builder jobConfig = generateLongRunJobConfig(numTasks);
    // Configuring participants
    setParticipantsCapacity(perNodeTaskLimitation);
    // schedule job1
    String jobName1 = "PriorityJob1";
    Workflow flow1 = WorkflowGenerator.generateSingleJobWorkflowBuilder(jobName1, jobConfig).build();
    _driver.start(flow1);
    _driver.pollForJobState(flow1.getName(), TaskUtil.getNamespacedJobName(flow1.getName(), jobName1), TaskState.IN_PROGRESS);
    // Wait for tasks to be picked up
    Thread.sleep(4000);
    Assert.assertEquals(countRunningPartition(flow1, jobName1), _numNodes * perNodeTaskLimitation);
    // schedule job2
    String jobName2 = "PriorityJob2";
    Workflow flow2 = WorkflowGenerator.generateSingleJobWorkflowBuilder(jobName2, jobConfig).build();
    _driver.start(flow2);
    _driver.pollForJobState(flow2.getName(), TaskUtil.getNamespacedJobName(flow2.getName(), jobName2), TaskState.IN_PROGRESS);
    // Wait for tasks to be picked up
    Thread.sleep(1500);
    Assert.assertEquals(countRunningPartition(flow2, jobName2), 0);
    // Increasing participants capacity
    perNodeTaskLimitation = 2 * perNodeTaskLimitation;
    setParticipantsCapacity(perNodeTaskLimitation);
    Thread.sleep(1500);
    // Additional capacity should all be used by job1
    Assert.assertEquals(countRunningPartition(flow1, jobName1), _numNodes * perNodeTaskLimitation);
    Assert.assertEquals(countRunningPartition(flow2, jobName2), 0);
    _driver.stop(flow1.getName());
    _driver.pollForWorkflowState(flow1.getName(), TaskState.STOPPED);
    _driver.stop(flow2.getName());
    _driver.pollForWorkflowState(flow2.getName(), TaskState.STOPPED);
}

Example 8 with Workflow

use of org.apache.helix.task.Workflow in project helix by apache.

the class TestTaskThrottling method testTaskThrottle.

@Test
public void testTaskThrottle() throws InterruptedException {
    int numTasks = 30 * _numNodes;
    int perNodeTaskLimitation = 5;
    JobConfig.Builder jobConfig = generateLongRunJobConfig(numTasks);
    // 1. Job executed in the participants with no limitation
    String jobName1 = "Job1";
    Workflow flow = WorkflowGenerator.generateSingleJobWorkflowBuilder(jobName1, jobConfig).build();
    _driver.start(flow);
    _driver.pollForJobState(flow.getName(), TaskUtil.getNamespacedJobName(flow.getName(), jobName1), TaskState.IN_PROGRESS);
    // Wait for tasks to be picked up
    Thread.sleep(1500);
    Assert.assertEquals(countRunningPartition(flow, jobName1), numTasks);
    _driver.stop(flow.getName());
    _driver.pollForWorkflowState(flow.getName(), TaskState.STOPPED);
    // 2. Job executed in the participants with max task limitation
    // Configuring cluster
    HelixConfigScope scope = new HelixConfigScopeBuilder(HelixConfigScope.ConfigScopeProperty.CLUSTER).forCluster(CLUSTER_NAME).build();
    Map<String, String> properties = new HashMap<String, String>();
    properties.put(ClusterConfig.ClusterConfigProperty.MAX_CONCURRENT_TASK_PER_INSTANCE.name(), new Integer(perNodeTaskLimitation).toString());
    _setupTool.getClusterManagementTool().setConfig(scope, properties);
    String jobName2 = "Job2";
    flow = WorkflowGenerator.generateSingleJobWorkflowBuilder(jobName2, jobConfig).build();
    _driver.start(flow);
    _driver.pollForJobState(flow.getName(), TaskUtil.getNamespacedJobName(flow.getName(), jobName2), TaskState.IN_PROGRESS);
    // Wait for tasks to be picked up
    Thread.sleep(4000);
    Assert.assertEquals(countRunningPartition(flow, jobName2), _numNodes * perNodeTaskLimitation);
    _driver.stop(flow.getName());
    _driver.pollForWorkflowState(flow.getName(), TaskState.STOPPED);
    // 3. Ensure job can finish normally
    jobConfig.setJobCommandConfigMap(ImmutableMap.of(MockTask.TIMEOUT_CONFIG, "10"));
    String jobName3 = "Job3";
    flow = WorkflowGenerator.generateSingleJobWorkflowBuilder(jobName3, jobConfig).build();
    _driver.start(flow);
    _driver.pollForJobState(flow.getName(), TaskUtil.getNamespacedJobName(flow.getName(), jobName3), TaskState.COMPLETED);
}

Example 9 with Workflow

use of org.apache.helix.task.Workflow in project helix by apache.

the class TestWorkflowJobDependency method testWorkflowWithDependencies.

@Test
public void testWorkflowWithDependencies() throws InterruptedException {
    String workflowName = TestHelper.getTestMethodName();
    final int PARALLEL_NUM = 2;
    // Workflow setup
    WorkflowConfig.Builder workflowcfgBuilder = new WorkflowConfig.Builder().setWorkflowId(workflowName).setParallelJobs(PARALLEL_NUM);
    Workflow.Builder builder = new Workflow.Builder(workflowName);
    builder.setWorkflowConfig(workflowcfgBuilder.build());
    builder.addParentChildDependency("job" + _testDbs.get(0), "job" + _testDbs.get(1));
    for (int i = 0; i < 2; i++) {
        JobConfig.Builder jobConfig = new JobConfig.Builder().setCommand(MockTask.TASK_COMMAND).setTargetResource(_testDbs.get(i)).setTargetPartitionStates(Sets.newHashSet("SLAVE", "MASTER")).setJobCommandConfigMap(WorkflowGenerator.DEFAULT_COMMAND_CONFIG);
        String jobName = "job" + _testDbs.get(i);
        builder.addJob(jobName, jobConfig);
    }
    // Start workflow
    Workflow workflow = builder.build();
    _driver.start(workflow);
    // Wait until the workflow completes
    _driver.pollForWorkflowState(workflowName, TaskState.COMPLETED);
    JobContext context1 = _driver.getJobContext(TaskUtil.getNamespacedJobName(workflowName, "job" + _testDbs.get(0)));
    JobContext context2 = _driver.getJobContext(TaskUtil.getNamespacedJobName(workflowName, "job" + _testDbs.get(1)));
    Assert.assertTrue(context2.getStartTime() - context1.getFinishTime() >= 0L);
}

Example 10 with Workflow

use of org.apache.helix.task.Workflow in project helix by apache.

the class TestJobTimeout method testTaskRunningIndefinitely.

@Test
public void testTaskRunningIndefinitely() throws InterruptedException {
    // first job runs indefinitely and timeout, the second job runs successfully, the workflow succeed.
    final String FIRST_JOB = "first_job";
    final String SECOND_JOB = "second_job";
    final String WORKFLOW_NAME = TestHelper.getTestMethodName();
    final String DB_NAME = WorkflowGenerator.DEFAULT_TGT_DB;
    JobConfig.Builder firstJobBuilder = new JobConfig.Builder().setWorkflow(WORKFLOW_NAME).setTargetResource(DB_NAME).setTargetPartitionStates(Sets.newHashSet(MasterSlaveSMD.States.MASTER.name())).setCommand(MockTask.TASK_COMMAND).setJobCommandConfigMap(// task stuck
    ImmutableMap.of(MockTask.TIMEOUT_CONFIG, "99999999")).setTimeout(1000);
    JobConfig.Builder secondJobBuilder = new JobConfig.Builder().setWorkflow(WORKFLOW_NAME).setTargetResource(DB_NAME).setTargetPartitionStates(Sets.newHashSet(MasterSlaveSMD.States.MASTER.name())).setCommand(MockTask.TASK_COMMAND).setIgnoreDependentJobFailure(// ignore first job's timeout
    true);
    WorkflowConfig.Builder workflowConfigBuilder = new WorkflowConfig.Builder(WORKFLOW_NAME).setFailureThreshold(// workflow ignores first job's timeout and schedule second job and succeed.
    1);
    Workflow.Builder workflowBuilder = new Workflow.Builder(WORKFLOW_NAME).setWorkflowConfig(workflowConfigBuilder.build()).addJob(FIRST_JOB, firstJobBuilder).addJob(SECOND_JOB, secondJobBuilder).addParentChildDependency(FIRST_JOB, SECOND_JOB);
    _driver.start(workflowBuilder.build());
    _driver.pollForJobState(WORKFLOW_NAME, TaskUtil.getNamespacedJobName(WORKFLOW_NAME, FIRST_JOB), TaskState.TIMED_OUT);
    _driver.pollForJobState(WORKFLOW_NAME, TaskUtil.getNamespacedJobName(WORKFLOW_NAME, SECOND_JOB), TaskState.COMPLETED);
    _driver.pollForWorkflowState(WORKFLOW_NAME, TaskState.COMPLETED);
    JobContext jobContext = _driver.getJobContext(TaskUtil.getNamespacedJobName(WORKFLOW_NAME, FIRST_JOB));
    for (int pId : jobContext.getPartitionSet()) {
        // All tasks aborted because of job timeout
        Assert.assertEquals(jobContext.getPartitionState(pId), TaskPartitionState.TASK_ABORTED);
    }
}