Examples with JobQueue org.apache.helix.task.JobQueue used on opensource projects

Example 6 with JobQueue

use of org.apache.helix.task.JobQueue in project helix by apache.

the class TestUpdateWorkflow method testUpdateStoppedQueue.

@Test
public void testUpdateStoppedQueue() throws InterruptedException {
    String queueName = TestHelper.getTestMethodName();
    // Create a queue
    LOG.info("Starting job-queue: " + queueName);
    JobQueue queue = createDefaultRecurrentJobQueue(queueName, 2);
    _driver.start(queue);
    WorkflowContext wCtx = TaskTestUtil.pollForWorkflowContext(_driver, queueName);
    // ensure current schedule is started
    String scheduledQueue = wCtx.getLastScheduledSingleWorkflow();
    _driver.pollForWorkflowState(scheduledQueue, TaskState.IN_PROGRESS);
    _driver.stop(queueName);
    WorkflowConfig workflowConfig = _driver.getWorkflowConfig(queueName);
    Assert.assertEquals(workflowConfig.getTargetState(), TargetState.STOP);
    WorkflowConfig.Builder configBuilder = new WorkflowConfig.Builder(workflowConfig);
    Calendar startTime = Calendar.getInstance();
    startTime.set(Calendar.SECOND, startTime.get(Calendar.SECOND) + 1);
    ScheduleConfig scheduleConfig = ScheduleConfig.recurringFromDate(startTime.getTime(), TimeUnit.MINUTES, 2);
    configBuilder.setScheduleConfig(scheduleConfig);
    _driver.updateWorkflow(queueName, configBuilder.build());
    workflowConfig = _driver.getWorkflowConfig(queueName);
    Assert.assertEquals(workflowConfig.getTargetState(), TargetState.STOP);
    _driver.resume(queueName);
    // ensure current schedule is completed
    _driver.pollForWorkflowState(scheduledQueue, TaskState.COMPLETED);
    Thread.sleep(1000);
    wCtx = TaskTestUtil.pollForWorkflowContext(_driver, queueName);
    scheduledQueue = wCtx.getLastScheduledSingleWorkflow();
    WorkflowConfig wCfg = _driver.getWorkflowConfig(scheduledQueue);
    Calendar configStartTime = Calendar.getInstance();
    configStartTime.setTime(wCfg.getStartTime());
    Assert.assertTrue((startTime.get(Calendar.HOUR_OF_DAY) == configStartTime.get(Calendar.HOUR_OF_DAY) && startTime.get(Calendar.MINUTE) == configStartTime.get(Calendar.MINUTE) && startTime.get(Calendar.SECOND) == configStartTime.get(Calendar.SECOND)));
}

Example 7 with JobQueue

use of org.apache.helix.task.JobQueue in project helix by apache.

the class TestTaskRebalancerParallel method testWhenDisallowOverlapJobAssignment.

/**
 * This test starts 4 jobs in job queue, the job all stuck, and verify that
 * (1) the number of running job does not exceed configured max allowed parallel jobs
 * (2) one instance can only be assigned to one job in the workflow
 */
@Test
public void testWhenDisallowOverlapJobAssignment() throws Exception {
    String queueName = TestHelper.getTestMethodName();
    WorkflowConfig.Builder cfgBuilder = new WorkflowConfig.Builder(queueName);
    cfgBuilder.setParallelJobs(PARALLEL_COUNT);
    cfgBuilder.setAllowOverlapJobAssignment(false);
    JobQueue.Builder queueBuild = new JobQueue.Builder(queueName).setWorkflowConfig(cfgBuilder.build());
    JobQueue queue = queueBuild.build();
    _driver.createQueue(queue);
    List<JobConfig.Builder> jobConfigBuilders = new ArrayList<JobConfig.Builder>();
    for (String testDbName : _testDbs) {
        jobConfigBuilders.add(new JobConfig.Builder().setCommand(MockTask.TASK_COMMAND).setTargetResource(testDbName).setTargetPartitionStates(Collections.singleton("SLAVE")));
    }
    _driver.stop(queueName);
    for (int i = 0; i < jobConfigBuilders.size(); ++i) {
        _driver.enqueueJob(queueName, "job_" + (i + 1), jobConfigBuilders.get(i));
    }
    _driver.resume(queueName);
    Thread.sleep(2000);
    Assert.assertTrue(TaskTestUtil.pollForWorkflowParallelState(_driver, queueName));
}

Example 8 with JobQueue

use of org.apache.helix.task.JobQueue in project helix by apache.

the class TestTaskRebalancerParallel method testWhenAllowOverlapJobAssignment.

/**
 * This test starts 4 jobs in job queue, the job all stuck, and verify that
 * (1) the number of running job does not exceed configured max allowed parallel jobs
 * (2) one instance can be assigned to multiple jobs in the workflow when allow overlap assignment
 */
@Test(dependsOnMethods = { "testWhenDisallowOverlapJobAssignment" })
public void testWhenAllowOverlapJobAssignment() throws Exception {
    // Disable all participants except one to enforce all assignment to be on one host
    for (int i = 1; i < _numNodes; i++) {
        _participants[i].syncStop();
    }
    ClusterLiveNodesVerifier verifier = new ClusterLiveNodesVerifier(_gZkClient, CLUSTER_NAME, Collections.singletonList(_participants[0].getInstanceName()));
    Assert.assertTrue(verifier.verify());
    String queueName = TestHelper.getTestMethodName();
    WorkflowConfig.Builder cfgBuilder = new WorkflowConfig.Builder(queueName);
    cfgBuilder.setParallelJobs(PARALLEL_COUNT);
    cfgBuilder.setAllowOverlapJobAssignment(true);
    JobQueue.Builder queueBuild = new JobQueue.Builder(queueName).setWorkflowConfig(cfgBuilder.build());
    JobQueue queue = queueBuild.build();
    _driver.createQueue(queue);
    // Create jobs that can be assigned to any instances
    List<JobConfig.Builder> jobConfigBuilders = new ArrayList<JobConfig.Builder>();
    for (int i = 0; i < PARALLEL_COUNT; i++) {
        List<TaskConfig> taskConfigs = new ArrayList<TaskConfig>();
        for (int j = 0; j < TASK_COUNT; j++) {
            taskConfigs.add(new TaskConfig.Builder().setTaskId("task_" + j).setCommand(MockTask.TASK_COMMAND).build());
        }
        jobConfigBuilders.add(new JobConfig.Builder().addTaskConfigs(taskConfigs));
    }
    _driver.stop(queueName);
    for (int i = 0; i < jobConfigBuilders.size(); ++i) {
        _driver.enqueueJob(queueName, "job_" + (i + 1), jobConfigBuilders.get(i));
    }
    _driver.resume(queueName);
    Thread.sleep(2000);
    Assert.assertTrue(TaskTestUtil.pollForWorkflowParallelState(_driver, queueName));
    for (int i = 1; i < _numNodes; i++) {
        _participants[i].syncStart();
    }
}

Example 9 with JobQueue

use of org.apache.helix.task.JobQueue in project helix by apache.

the class TestTaskRebalancerStopResume method stopAndResumeNamedQueue.

@Test
public void stopAndResumeNamedQueue() throws Exception {
    String queueName = TestHelper.getTestMethodName();
    // Create a queue
    LOG.info("Starting job-queue: " + queueName);
    JobQueue queue = new JobQueue.Builder(queueName).build();
    _driver.createQueue(queue);
    // Enqueue jobs
    Set<String> master = Sets.newHashSet("MASTER");
    JobConfig.Builder job1 = new JobConfig.Builder().setCommand(MockTask.TASK_COMMAND).setTargetResource(WorkflowGenerator.DEFAULT_TGT_DB).setTargetPartitionStates(master);
    String job1Name = "masterJob";
    LOG.info("Enqueuing job: " + job1Name);
    _driver.enqueueJob(queueName, job1Name, job1);
    Set<String> slave = Sets.newHashSet("SLAVE");
    JobConfig.Builder job2 = new JobConfig.Builder().setCommand(MockTask.TASK_COMMAND).setTargetResource(WorkflowGenerator.DEFAULT_TGT_DB).setTargetPartitionStates(slave);
    String job2Name = "slaveJob";
    LOG.info("Enqueuing job: " + job2Name);
    _driver.enqueueJob(queueName, job2Name, job2);
    String namespacedJob1 = String.format("%s_%s", queueName, job1Name);
    _driver.pollForJobState(queueName, namespacedJob1, TaskState.IN_PROGRESS);
    // stop job1
    LOG.info("Pausing job-queue: " + queueName);
    _driver.stop(queueName);
    _driver.pollForJobState(queueName, namespacedJob1, TaskState.STOPPED);
    _driver.pollForWorkflowState(queueName, TaskState.STOPPED);
    // Ensure job2 is not started
    TimeUnit.MILLISECONDS.sleep(200);
    String namespacedJob2 = String.format("%s_%s", queueName, job2Name);
    TaskTestUtil.pollForEmptyJobState(_driver, queueName, job2Name);
    LOG.info("Resuming job-queue: " + queueName);
    _driver.resume(queueName);
    // Ensure successful completion
    _driver.pollForJobState(queueName, namespacedJob1, TaskState.COMPLETED);
    _driver.pollForJobState(queueName, namespacedJob2, TaskState.COMPLETED);
    JobContext masterJobContext = _driver.getJobContext(namespacedJob1);
    JobContext slaveJobContext = _driver.getJobContext(namespacedJob2);
    // Ensure correct ordering
    long job1Finish = masterJobContext.getFinishTime();
    long job2Start = slaveJobContext.getStartTime();
    Assert.assertTrue(job2Start >= job1Finish);
    // Flush queue and check cleanup
    LOG.info("Flusing job-queue: " + queueName);
    _driver.flushQueue(queueName);
    verifyJobDeleted(queueName, namespacedJob1);
    verifyJobDeleted(queueName, namespacedJob2);
    verifyJobNotInQueue(queueName, namespacedJob1);
    verifyJobNotInQueue(queueName, namespacedJob2);
}

Example 10 with JobQueue

use of org.apache.helix.task.JobQueue in project helix by apache.

the class TestUpdateWorkflow method testUpdateRunningQueue.

@Test
public void testUpdateRunningQueue() throws InterruptedException {
    String queueName = TestHelper.getTestMethodName();
    // Create a queue
    LOG.info("Starting job-queue: " + queueName);
    JobQueue queue = createDefaultRecurrentJobQueue(queueName, 2);
    _driver.start(queue);
    WorkflowContext wCtx = TaskTestUtil.pollForWorkflowContext(_driver, queueName);
    WorkflowConfig workflowConfig = _driver.getWorkflowConfig(queueName);
    WorkflowConfig.Builder configBuilder = new WorkflowConfig.Builder(workflowConfig);
    Calendar startTime = Calendar.getInstance();
    startTime.set(Calendar.SECOND, startTime.get(Calendar.SECOND) + 1);
    ScheduleConfig scheduleConfig = ScheduleConfig.recurringFromDate(startTime.getTime(), TimeUnit.MINUTES, 2);
    configBuilder.setScheduleConfig(scheduleConfig);
    // ensure current schedule is started
    String scheduledQueue = wCtx.getLastScheduledSingleWorkflow();
    _driver.pollForWorkflowState(scheduledQueue, TaskState.IN_PROGRESS);
    _driver.updateWorkflow(queueName, configBuilder.build());
    // ensure current schedule is completed
    _driver.pollForWorkflowState(scheduledQueue, TaskState.COMPLETED);
    Thread.sleep(1000);
    wCtx = TaskTestUtil.pollForWorkflowContext(_driver, queueName);
    scheduledQueue = wCtx.getLastScheduledSingleWorkflow();
    WorkflowConfig wCfg = _driver.getWorkflowConfig(scheduledQueue);
    Calendar configStartTime = Calendar.getInstance();
    configStartTime.setTime(wCfg.getStartTime());
    Assert.assertTrue((startTime.get(Calendar.HOUR_OF_DAY) == configStartTime.get(Calendar.HOUR_OF_DAY) && startTime.get(Calendar.MINUTE) == configStartTime.get(Calendar.MINUTE) && startTime.get(Calendar.SECOND) == configStartTime.get(Calendar.SECOND)));
}