Examples with WorkflowConfig org.apache.helix.task.WorkflowConfig used on opensource projects

Example 16 with WorkflowConfig

use of org.apache.helix.task.WorkflowConfig in project incubator-gobblin by apache.

the class GobblinHelixTaskDriver method deleteWorkflow.

/**
 * Delete the workflow
 *
 * @param workflow  The workflow name
 * @param timeout   The timeout for deleting the workflow/queue in seconds
 */
public void deleteWorkflow(String workflow, long timeout) throws InterruptedException {
    WorkflowConfig workflowConfig = _taskDriver.getWorkflowConfig(workflow);
    // set the target state if not already set
    if (workflowConfig != null && workflowConfig.getTargetState() != TargetState.DELETE) {
        _taskDriver.delete(workflow);
    }
    long endTime = System.currentTimeMillis() + (timeout * 1000);
    // check for completion of deletion request
    while (System.currentTimeMillis() <= endTime) {
        WorkflowContext workflowContext = _taskDriver.getWorkflowContext(workflow);
        if (workflowContext != null) {
            Thread.sleep(1000);
        } else {
            // Successfully deleted
            return;
        }
    }
    // Failed to complete deletion within timeout
    throw new HelixException(String.format("Fail to delete the workflow/queue %s within %d seconds.", workflow, timeout));
}

Example 17 with WorkflowConfig

use of org.apache.helix.task.WorkflowConfig in project helix by apache.

the class TaskTestUtil method pollForWorkflowParallelState.

// 1. Different jobs in a same work flow is in RUNNING at the same time
// 2. When disallow overlap assignment, no two jobs in the same work flow is in RUNNING at the same instance
// Use this method with caution because it assumes workflow doesn't finish too quickly and number of parallel running
// tasks can be counted.
public static boolean pollForWorkflowParallelState(TaskDriver driver, String workflowName) throws InterruptedException {
    WorkflowConfig workflowConfig = driver.getWorkflowConfig(workflowName);
    Assert.assertNotNull(workflowConfig);
    WorkflowContext workflowContext = null;
    while (workflowContext == null) {
        workflowContext = driver.getWorkflowContext(workflowName);
        Thread.sleep(100);
    }
    int maxRunningCount = 0;
    boolean finished = false;
    while (!finished) {
        finished = true;
        int runningCount = 0;
        workflowContext = driver.getWorkflowContext(workflowName);
        for (String jobName : workflowConfig.getJobDag().getAllNodes()) {
            TaskState jobState = workflowContext.getJobState(jobName);
            if (jobState == TaskState.IN_PROGRESS) {
                ++runningCount;
                finished = false;
            }
        }
        if (runningCount > maxRunningCount) {
            maxRunningCount = runningCount;
        }
        List<JobContext> jobContextList = new ArrayList<JobContext>();
        for (String jobName : workflowConfig.getJobDag().getAllNodes()) {
            JobContext jobContext = driver.getJobContext(jobName);
            if (jobContext != null) {
                jobContextList.add(driver.getJobContext(jobName));
            }
        }
        if (!workflowConfig.isAllowOverlapJobAssignment()) {
            Set<String> instances = new HashSet<String>();
            for (JobContext jobContext : jobContextList) {
                for (int partition : jobContext.getPartitionSet()) {
                    String instance = jobContext.getAssignedParticipant(partition);
                    TaskPartitionState taskPartitionState = jobContext.getPartitionState(partition);
                    if (instance == null) {
                        continue;
                    }
                    if (taskPartitionState != TaskPartitionState.INIT && taskPartitionState != TaskPartitionState.RUNNING) {
                        continue;
                    }
                    if (instances.contains(instance)) {
                        return false;
                    }
                    TaskPartitionState state = jobContext.getPartitionState(partition);
                    if (state != TaskPartitionState.COMPLETED) {
                        instances.add(instance);
                    }
                }
            }
        }
        Thread.sleep(100);
    }
    return maxRunningCount > 1 && (workflowConfig.isJobQueue() ? maxRunningCount <= workflowConfig.getParallelJobs() : true);
}

Example 18 with WorkflowConfig

use of org.apache.helix.task.WorkflowConfig in project helix by apache.

the class TestTaskRebalancerStopResume method verifyJobNotInQueue.

private void verifyJobNotInQueue(String queueName, String namedSpacedJobName) {
    WorkflowConfig workflowCfg = _driver.getWorkflowConfig(queueName);
    JobDag dag = workflowCfg.getJobDag();
    Assert.assertFalse(dag.getAllNodes().contains(namedSpacedJobName));
    Assert.assertFalse(dag.getChildrenToParents().containsKey(namedSpacedJobName));
    Assert.assertFalse(dag.getParentsToChildren().containsKey(namedSpacedJobName));
}

Example 19 with WorkflowConfig

use of org.apache.helix.task.WorkflowConfig in project helix by apache.

the class TestUpdateWorkflow method testUpdateRunningQueue.

@Test
public void testUpdateRunningQueue() throws InterruptedException {
    String queueName = TestHelper.getTestMethodName();
    // Create a queue
    LOG.info("Starting job-queue: " + queueName);
    JobQueue queue = createDefaultRecurrentJobQueue(queueName, 2);
    _driver.start(queue);
    WorkflowContext wCtx = TaskTestUtil.pollForWorkflowContext(_driver, queueName);
    WorkflowConfig workflowConfig = _driver.getWorkflowConfig(queueName);
    WorkflowConfig.Builder configBuilder = new WorkflowConfig.Builder(workflowConfig);
    Calendar startTime = Calendar.getInstance();
    startTime.set(Calendar.SECOND, startTime.get(Calendar.SECOND) + 1);
    ScheduleConfig scheduleConfig = ScheduleConfig.recurringFromDate(startTime.getTime(), TimeUnit.MINUTES, 2);
    configBuilder.setScheduleConfig(scheduleConfig);
    // ensure current schedule is started
    String scheduledQueue = wCtx.getLastScheduledSingleWorkflow();
    _driver.pollForWorkflowState(scheduledQueue, TaskState.IN_PROGRESS);
    _driver.updateWorkflow(queueName, configBuilder.build());
    // ensure current schedule is completed
    _driver.pollForWorkflowState(scheduledQueue, TaskState.COMPLETED);
    Thread.sleep(1000);
    wCtx = TaskTestUtil.pollForWorkflowContext(_driver, queueName);
    scheduledQueue = wCtx.getLastScheduledSingleWorkflow();
    WorkflowConfig wCfg = _driver.getWorkflowConfig(scheduledQueue);
    Calendar configStartTime = Calendar.getInstance();
    configStartTime.setTime(wCfg.getStartTime());
    Assert.assertTrue((startTime.get(Calendar.HOUR_OF_DAY) == configStartTime.get(Calendar.HOUR_OF_DAY) && startTime.get(Calendar.MINUTE) == configStartTime.get(Calendar.MINUTE) && startTime.get(Calendar.SECOND) == configStartTime.get(Calendar.SECOND)));
}

Example 20 with WorkflowConfig

use of org.apache.helix.task.WorkflowConfig in project helix by apache.

the class TestRecurringJobQueue method testDeletingRecurrentQueueWithHistory.

@Test
public void testDeletingRecurrentQueueWithHistory() throws Exception {
    final String queueName = TestHelper.getTestMethodName();
    // Create a queue
    LOG.info("Starting job-queue: " + queueName);
    JobQueue.Builder queueBuild = TaskTestUtil.buildRecurrentJobQueue(queueName, 0, 60, TargetState.STOP);
    createAndEnqueueJob(queueBuild, 2);
    _driver.createQueue(queueBuild.build());
    WorkflowConfig workflowConfig = _driver.getWorkflowConfig(queueName);
    Assert.assertEquals(workflowConfig.getTargetState(), TargetState.STOP);
    _driver.resume(queueName);
    WorkflowContext wCtx;
    // wait until at least 2 workflows are scheduled based on template queue
    do {
        Thread.sleep(60000);
        wCtx = TaskTestUtil.pollForWorkflowContext(_driver, queueName);
    } while (wCtx.getScheduledWorkflows().size() < 2);
    // Stop recurring workflow
    _driver.stop(queueName);
    _driver.pollForWorkflowState(queueName, TaskState.STOPPED);
    // Record all scheduled workflows
    wCtx = TaskTestUtil.pollForWorkflowContext(_driver, queueName);
    List<String> scheduledWorkflows = new ArrayList<String>(wCtx.getScheduledWorkflows());
    final String lastScheduledWorkflow = wCtx.getLastScheduledSingleWorkflow();
    // Delete recurrent workflow
    _driver.delete(queueName);
    // Wait until recurrent workflow and the last scheduled workflow are cleaned up
    boolean result = TestHelper.verify(new TestHelper.Verifier() {

        @Override
        public boolean verify() throws Exception {
            WorkflowContext wCtx = _driver.getWorkflowContext(queueName);
            WorkflowContext lastWfCtx = _driver.getWorkflowContext(lastScheduledWorkflow);
            return (wCtx == null && lastWfCtx == null);
        }
    }, 5 * 1000);
    Assert.assertTrue(result);
    for (String scheduledWorkflow : scheduledWorkflows) {
        WorkflowContext scheduledWorkflowCtx = _driver.getWorkflowContext(scheduledWorkflow);
        WorkflowConfig scheduledWorkflowCfg = _driver.getWorkflowConfig(scheduledWorkflow);
        Assert.assertNull(scheduledWorkflowCtx);
        Assert.assertNull(scheduledWorkflowCfg);
    }
}