Examples with JobContext org.apache.helix.task.JobContext used on opensource projects

Example 6 with JobContext

use of org.apache.helix.task.JobContext in project helix by apache.

the class TestTaskRebalancerStopResume method stopDeleteJobAndResumeNamedQueue.

@Test
public void stopDeleteJobAndResumeNamedQueue() throws Exception {
    String queueName = TestHelper.getTestMethodName();
    // Create a queue
    LOG.info("Starting job-queue: " + queueName);
    JobQueue.Builder queueBuilder = TaskTestUtil.buildJobQueue(queueName);
    // Create and Enqueue jobs
    List<String> currentJobNames = new ArrayList<String>();
    for (int i = 0; i <= 4; i++) {
        String targetPartition = (i == 0) ? "MASTER" : "SLAVE";
        JobConfig.Builder jobBuilder = new JobConfig.Builder().setCommand(MockTask.TASK_COMMAND).setTargetResource(WorkflowGenerator.DEFAULT_TGT_DB).setTargetPartitionStates(Sets.newHashSet(targetPartition));
        String jobName = targetPartition.toLowerCase() + "Job" + i;
        LOG.info("Enqueuing job: " + jobName);
        queueBuilder.enqueueJob(jobName, jobBuilder);
        currentJobNames.add(i, jobName);
    }
    _driver.createQueue(queueBuilder.build());
    // ensure job 1 is started before deleting it
    String deletedJob1 = currentJobNames.get(0);
    String namedSpaceDeletedJob1 = String.format("%s_%s", queueName, deletedJob1);
    _driver.pollForJobState(queueName, namedSpaceDeletedJob1, TaskState.IN_PROGRESS);
    // stop the queue
    LOG.info("Pausing job-queue: " + queueName);
    _driver.stop(queueName);
    _driver.pollForJobState(queueName, namedSpaceDeletedJob1, TaskState.STOPPED);
    _driver.pollForWorkflowState(queueName, TaskState.STOPPED);
    // delete the in-progress job (job 1) and verify it being deleted
    _driver.deleteJob(queueName, deletedJob1);
    verifyJobDeleted(queueName, namedSpaceDeletedJob1);
    LOG.info("Resuming job-queue: " + queueName);
    _driver.resume(queueName);
    // ensure job 2 is started
    _driver.pollForJobState(queueName, String.format("%s_%s", queueName, currentJobNames.get(1)), TaskState.IN_PROGRESS);
    // stop the queue
    LOG.info("Pausing job-queue: " + queueName);
    _driver.stop(queueName);
    _driver.pollForJobState(queueName, String.format("%s_%s", queueName, currentJobNames.get(1)), TaskState.STOPPED);
    _driver.pollForWorkflowState(queueName, TaskState.STOPPED);
    // Ensure job 3 is not started before deleting it
    String deletedJob2 = currentJobNames.get(2);
    String namedSpaceDeletedJob2 = String.format("%s_%s", queueName, deletedJob2);
    TaskTestUtil.pollForEmptyJobState(_driver, queueName, namedSpaceDeletedJob2);
    // delete not-started job (job 3) and verify it being deleted
    _driver.deleteJob(queueName, deletedJob2);
    verifyJobDeleted(queueName, namedSpaceDeletedJob2);
    LOG.info("Resuming job-queue: " + queueName);
    _driver.resume(queueName);
    currentJobNames.remove(deletedJob1);
    currentJobNames.remove(deletedJob2);
    // add job 3 back
    JobConfig.Builder job = new JobConfig.Builder().setCommand(MockTask.TASK_COMMAND).setTargetResource(WorkflowGenerator.DEFAULT_TGT_DB).setTargetPartitionStates(Sets.newHashSet("SLAVE"));
    LOG.info("Enqueuing job: " + deletedJob2);
    _driver.enqueueJob(queueName, deletedJob2, job);
    currentJobNames.add(deletedJob2);
    // Ensure the jobs left are successful completed in the correct order
    long preJobFinish = 0;
    for (int i = 0; i < currentJobNames.size(); i++) {
        String namedSpaceJobName = String.format("%s_%s", queueName, currentJobNames.get(i));
        _driver.pollForJobState(queueName, namedSpaceJobName, TaskState.COMPLETED);
        JobContext jobContext = _driver.getJobContext(namedSpaceJobName);
        long jobStart = jobContext.getStartTime();
        Assert.assertTrue(jobStart >= preJobFinish);
        preJobFinish = jobContext.getFinishTime();
    }
    TimeUnit.MILLISECONDS.sleep(2000);
    // Flush queue
    LOG.info("Flusing job-queue: " + queueName);
    _driver.flushQueue(queueName);
    // TODO: Use TestHelper.verify() instead of waiting here.
    TimeUnit.MILLISECONDS.sleep(2000);
    // verify the cleanup
    for (int i = 0; i < currentJobNames.size(); i++) {
        String namedSpaceJobName = String.format("%s_%s", queueName, currentJobNames.get(i));
        verifyJobDeleted(queueName, namedSpaceJobName);
        verifyJobNotInQueue(queueName, namedSpaceJobName);
    }
}

Example 7 with JobContext

use of org.apache.helix.task.JobContext in project helix by apache.

the class TestTaskThrottling method countRunningPartition.

private int countRunningPartition(Workflow flow, String jobName) {
    int runningPartition = 0;
    JobContext jobContext = _driver.getJobContext(TaskUtil.getNamespacedJobName(flow.getName(), jobName));
    for (int partition : jobContext.getPartitionSet()) {
        if (jobContext.getPartitionState(partition) != null && jobContext.getPartitionState(partition).equals(TaskPartitionState.RUNNING)) {
            runningPartition++;
        }
    }
    return runningPartition;
}

Example 8 with JobContext

use of org.apache.helix.task.JobContext in project helix by apache.

the class TestWorkflowJobDependency method testWorkflowWithDependencies.

@Test
public void testWorkflowWithDependencies() throws InterruptedException {
    String workflowName = TestHelper.getTestMethodName();
    final int PARALLEL_NUM = 2;
    // Workflow setup
    WorkflowConfig.Builder workflowcfgBuilder = new WorkflowConfig.Builder().setWorkflowId(workflowName).setParallelJobs(PARALLEL_NUM);
    Workflow.Builder builder = new Workflow.Builder(workflowName);
    builder.setWorkflowConfig(workflowcfgBuilder.build());
    builder.addParentChildDependency("job" + _testDbs.get(0), "job" + _testDbs.get(1));
    for (int i = 0; i < 2; i++) {
        JobConfig.Builder jobConfig = new JobConfig.Builder().setCommand(MockTask.TASK_COMMAND).setTargetResource(_testDbs.get(i)).setTargetPartitionStates(Sets.newHashSet("SLAVE", "MASTER")).setJobCommandConfigMap(WorkflowGenerator.DEFAULT_COMMAND_CONFIG);
        String jobName = "job" + _testDbs.get(i);
        builder.addJob(jobName, jobConfig);
    }
    // Start workflow
    Workflow workflow = builder.build();
    _driver.start(workflow);
    // Wait until the workflow completes
    _driver.pollForWorkflowState(workflowName, TaskState.COMPLETED);
    JobContext context1 = _driver.getJobContext(TaskUtil.getNamespacedJobName(workflowName, "job" + _testDbs.get(0)));
    JobContext context2 = _driver.getJobContext(TaskUtil.getNamespacedJobName(workflowName, "job" + _testDbs.get(1)));
    Assert.assertTrue(context2.getStartTime() - context1.getFinishTime() >= 0L);
}

Example 9 with JobContext

use of org.apache.helix.task.JobContext in project helix by apache.

the class TaskAdmin method list.

private static void list(TaskDriver taskDriver, String workflow) {
    WorkflowConfig wCfg = taskDriver.getWorkflowConfig(workflow);
    if (wCfg == null) {
        LOG.error("Workflow " + workflow + " does not exist!");
        return;
    }
    WorkflowContext wCtx = taskDriver.getWorkflowContext(workflow);
    LOG.info("Workflow " + workflow + " consists of the following tasks: " + wCfg.getJobDag().getAllNodes());
    String workflowState = (wCtx != null) ? wCtx.getWorkflowState().name() : TaskState.NOT_STARTED.name();
    LOG.info("Current state of workflow is " + workflowState);
    LOG.info("Job states are: ");
    LOG.info("-------");
    for (String job : wCfg.getJobDag().getAllNodes()) {
        TaskState jobState = (wCtx != null) ? wCtx.getJobState(job) : TaskState.NOT_STARTED;
        LOG.info("Job " + job + " is " + jobState);
        // fetch job information
        JobConfig jCfg = taskDriver.getJobConfig(job);
        JobContext jCtx = taskDriver.getJobContext(job);
        if (jCfg == null || jCtx == null) {
            LOG.info("-------");
            continue;
        }
        // calculate taskPartitions
        List<Integer> partitions = Lists.newArrayList(jCtx.getPartitionSet());
        Collections.sort(partitions);
        // report status
        for (Integer partition : partitions) {
            String taskId = jCtx.getTaskIdForPartition(partition);
            taskId = (taskId != null) ? taskId : jCtx.getTargetForPartition(partition);
            LOG.info("Task: " + taskId);
            TaskConfig taskConfig = jCfg.getTaskConfig(taskId);
            if (taskConfig != null) {
                LOG.info("Configuration: " + taskConfig.getConfigMap());
            }
            TaskPartitionState state = jCtx.getPartitionState(partition);
            state = (state != null) ? state : TaskPartitionState.INIT;
            LOG.info("State: " + state);
            String assignedParticipant = jCtx.getAssignedParticipant(partition);
            if (assignedParticipant != null) {
                LOG.info("Assigned participant: " + assignedParticipant);
            }
            LOG.info("-------");
        }
        LOG.info("-------");
    }
}

Example 10 with JobContext

use of org.apache.helix.task.JobContext in project helix by apache.

the class TaskTestUtil method buildJobContext.

public static JobContext buildJobContext(Long startTime, Long finishTime, TaskPartitionState... partitionStates) {
    JobContext jobContext = new JobContext(new ZNRecord(TaskUtil.TASK_CONTEXT_KW));
    jobContext.setStartTime(startTime == null ? System.currentTimeMillis() : startTime);
    jobContext.setFinishTime(finishTime == null ? System.currentTimeMillis() : finishTime);
    int partitionId = 0;
    for (TaskPartitionState partitionState : partitionStates) {
        jobContext.setPartitionState(partitionId++, partitionState);
    }
    return jobContext;
}