Examples with WorkflowContext org.apache.helix.task.WorkflowContext used on opensource projects

Example 16 with WorkflowContext

use of org.apache.helix.task.WorkflowContext in project incubator-gobblin by apache.

the class GobblinHelixTaskDriver method deleteWorkflow.

/**
 * Delete the workflow
 *
 * @param workflow  The workflow name
 * @param timeout   The timeout for deleting the workflow/queue in seconds
 */
public void deleteWorkflow(String workflow, long timeout) throws InterruptedException {
    WorkflowConfig workflowConfig = _taskDriver.getWorkflowConfig(workflow);
    // set the target state if not already set
    if (workflowConfig != null && workflowConfig.getTargetState() != TargetState.DELETE) {
        _taskDriver.delete(workflow);
    }
    long endTime = System.currentTimeMillis() + (timeout * 1000);
    // check for completion of deletion request
    while (System.currentTimeMillis() <= endTime) {
        WorkflowContext workflowContext = _taskDriver.getWorkflowContext(workflow);
        if (workflowContext != null) {
            Thread.sleep(1000);
        } else {
            // Successfully deleted
            return;
        }
    }
    // Failed to complete deletion within timeout
    throw new HelixException(String.format("Fail to delete the workflow/queue %s within %d seconds.", workflow, timeout));
}

Example 17 with WorkflowContext

use of org.apache.helix.task.WorkflowContext in project helix by apache.

the class TaskTestUtil method pollForWorkflowContext.

public static WorkflowContext pollForWorkflowContext(TaskDriver driver, String workflowResource) throws InterruptedException {
    // Wait for completion.
    long st = System.currentTimeMillis();
    WorkflowContext ctx;
    do {
        ctx = driver.getWorkflowContext(workflowResource);
        Thread.sleep(100);
    } while (ctx == null && System.currentTimeMillis() < st + _default_timeout);
    Assert.assertNotNull(ctx);
    return ctx;
}

Example 18 with WorkflowContext

use of org.apache.helix.task.WorkflowContext in project helix by apache.

the class TaskTestUtil method pollForWorkflowParallelState.

// 1. Different jobs in a same work flow is in RUNNING at the same time
// 2. When disallow overlap assignment, no two jobs in the same work flow is in RUNNING at the same instance
// Use this method with caution because it assumes workflow doesn't finish too quickly and number of parallel running
// tasks can be counted.
public static boolean pollForWorkflowParallelState(TaskDriver driver, String workflowName) throws InterruptedException {
    WorkflowConfig workflowConfig = driver.getWorkflowConfig(workflowName);
    Assert.assertNotNull(workflowConfig);
    WorkflowContext workflowContext = null;
    while (workflowContext == null) {
        workflowContext = driver.getWorkflowContext(workflowName);
        Thread.sleep(100);
    }
    int maxRunningCount = 0;
    boolean finished = false;
    while (!finished) {
        finished = true;
        int runningCount = 0;
        workflowContext = driver.getWorkflowContext(workflowName);
        for (String jobName : workflowConfig.getJobDag().getAllNodes()) {
            TaskState jobState = workflowContext.getJobState(jobName);
            if (jobState == TaskState.IN_PROGRESS) {
                ++runningCount;
                finished = false;
            }
        }
        if (runningCount > maxRunningCount) {
            maxRunningCount = runningCount;
        }
        List<JobContext> jobContextList = new ArrayList<JobContext>();
        for (String jobName : workflowConfig.getJobDag().getAllNodes()) {
            JobContext jobContext = driver.getJobContext(jobName);
            if (jobContext != null) {
                jobContextList.add(driver.getJobContext(jobName));
            }
        }
        if (!workflowConfig.isAllowOverlapJobAssignment()) {
            Set<String> instances = new HashSet<String>();
            for (JobContext jobContext : jobContextList) {
                for (int partition : jobContext.getPartitionSet()) {
                    String instance = jobContext.getAssignedParticipant(partition);
                    TaskPartitionState taskPartitionState = jobContext.getPartitionState(partition);
                    if (instance == null) {
                        continue;
                    }
                    if (taskPartitionState != TaskPartitionState.INIT && taskPartitionState != TaskPartitionState.RUNNING) {
                        continue;
                    }
                    if (instances.contains(instance)) {
                        return false;
                    }
                    TaskPartitionState state = jobContext.getPartitionState(partition);
                    if (state != TaskPartitionState.COMPLETED) {
                        instances.add(instance);
                    }
                }
            }
        }
        Thread.sleep(100);
    }
    return maxRunningCount > 1 && (workflowConfig.isJobQueue() ? maxRunningCount <= workflowConfig.getParallelJobs() : true);
}

Example 19 with WorkflowContext

use of org.apache.helix.task.WorkflowContext in project helix by apache.

the class TaskTestUtil method buildWorkflowContext.

public static WorkflowContext buildWorkflowContext(String workflowResource, TaskState workflowState, Long startTime, TaskState... jobStates) {
    WorkflowContext workflowContext = new WorkflowContext(new ZNRecord(TaskUtil.WORKFLOW_CONTEXT_KW));
    workflowContext.setName(workflowResource);
    workflowContext.setStartTime(startTime == null ? System.currentTimeMillis() : startTime);
    int jobId = 0;
    for (TaskState jobstate : jobStates) {
        workflowContext.setJobState(TaskUtil.getNamespacedJobName(workflowResource, JOB_KW) + jobId++, jobstate);
    }
    workflowContext.setWorkflowState(workflowState);
    return workflowContext;
}

Example 20 with WorkflowContext

use of org.apache.helix.task.WorkflowContext in project helix by apache.

the class TestTaskRebalancer method partitionSet.

@Test
public void partitionSet() throws Exception {
    final String jobResource = "partitionSet";
    ImmutableList<String> targetPartitions = ImmutableList.of("TestDB_1", "TestDB_2", "TestDB_3", "TestDB_5", "TestDB_8", "TestDB_13");
    // construct and submit our basic workflow
    Map<String, String> commandConfig = ImmutableMap.of(TIMEOUT_CONFIG, String.valueOf(100));
    JobConfig.Builder jobBuilder = JobConfig.Builder.fromMap(WorkflowGenerator.DEFAULT_JOB_CONFIG);
    jobBuilder.setJobCommandConfigMap(commandConfig).setMaxAttemptsPerTask(1).setTargetPartitions(targetPartitions);
    Workflow flow = WorkflowGenerator.generateSingleJobWorkflowBuilder(jobResource, jobBuilder).build();
    _driver.start(flow);
    // wait for job completeness/timeout
    _driver.pollForWorkflowState(jobResource, TaskState.COMPLETED);
    // see if resulting context completed successfully for our partition set
    String namespacedName = TaskUtil.getNamespacedJobName(jobResource);
    JobContext ctx = _driver.getJobContext(namespacedName);
    WorkflowContext workflowContext = _driver.getWorkflowContext(jobResource);
    Assert.assertNotNull(ctx);
    Assert.assertNotNull(workflowContext);
    Assert.assertEquals(workflowContext.getJobState(namespacedName), TaskState.COMPLETED);
    for (String pName : targetPartitions) {
        int i = ctx.getPartitionsByTarget().get(pName).get(0);
        Assert.assertEquals(ctx.getPartitionState(i), TaskPartitionState.COMPLETED);
        Assert.assertEquals(ctx.getPartitionNumAttempts(i), 1);
    }
}