Examples with WorkflowExecution alluxio.job.workflow.WorkflowExecution used on opensource projects

Example 1 with WorkflowExecution

use of alluxio.job.workflow.WorkflowExecution in project alluxio by Alluxio.

the class WorkflowTracker method getStatus.

/**
 * Gets information of the given job id.
 *
 * @param jobId the id of the job
 * @param verbose whether the output should be verbose
 * @return null if the job id isn't know by the workflow tracker. WorkflowInfo otherwise
 */
public WorkflowInfo getStatus(long jobId, boolean verbose) {
    WorkflowExecution workflowExecution = mWorkflows.get(jobId);
    if (workflowExecution == null) {
        return null;
    }
    ArrayList<Long> children = Lists.newArrayList(mChildren.get(jobId).iterator());
    Collections.sort(children);
    List<JobInfo> jobInfos = Lists.newArrayList();
    if (verbose) {
        for (long child : children) {
            try {
                jobInfos.add(mJobMaster.getStatus(child));
            } catch (JobDoesNotExistException e) {
                LOG.info(String.format("No job info on child job id %s. Skipping", child));
            }
        }
    }
    WorkflowInfo workflowInfo = new WorkflowInfo(jobId, workflowExecution.getName(), workflowExecution.getStatus(), workflowExecution.getLastUpdated(), workflowExecution.getErrorType(), workflowExecution.getErrorMessage(), jobInfos);
    return workflowInfo;
}

Example 2 with WorkflowExecution

use of alluxio.job.workflow.WorkflowExecution in project alluxio by Alluxio.

the class WorkflowTracker method stop.

private synchronized void stop(long jobId, Status status, String errorType, String errorMessage) {
    Long parentJobId = mParentWorkflow.get(jobId);
    if (parentJobId == null) {
        return;
    }
    WorkflowExecution workflowExecution = mWorkflows.get(parentJobId);
    workflowExecution.stop(status, errorType, errorMessage);
    stop(parentJobId, status, errorType, errorMessage);
    return;
}

Example 3 with WorkflowExecution

use of alluxio.job.workflow.WorkflowExecution in project alluxio by Alluxio.

the class WorkflowTracker method clean.

private synchronized void clean(long jobId) {
    mWorkflows.remove(jobId);
    mWaitingOn.remove(jobId);
    mChildren.remove(jobId);
    Long parentId = mParentWorkflow.remove(jobId);
    if (parentId == null) {
        return;
    }
    // TODO(bradley): share details of the child job to the parent workflow before deleting.
    ConcurrentHashSet<Long> siblings = mChildren.get(parentId);
    siblings.remove(jobId);
    if (siblings.isEmpty()) {
        WorkflowExecution parentExecution = mWorkflows.get(parentId);
        if (parentExecution != null && parentExecution.getStatus().isFinished()) {
            clean(parentId);
        }
    }
}

Example 4 with WorkflowExecution

use of alluxio.job.workflow.WorkflowExecution in project alluxio by Alluxio.

the class WorkflowTracker method run.

/**
 * Runs a workflow with the given configuration and job id.
 *
 * @param workflowConfig the workflow configuration
 * @param jobId the job id
 */
public synchronized void run(WorkflowConfig workflowConfig, long jobId) throws JobDoesNotExistException, ResourceExhaustedException {
    WorkflowExecution workflowExecution = WorkflowExecutionRegistry.INSTANCE.getExecution(workflowConfig);
    mWorkflows.put(jobId, workflowExecution);
    next(jobId);
}

Example 5 with WorkflowExecution

use of alluxio.job.workflow.WorkflowExecution in project alluxio by Alluxio.

the class WorkflowTracker method next.

private synchronized void next(long jobId) {
    WorkflowExecution workflowExecution = mWorkflows.get(jobId);
    mChildren.putIfAbsent(jobId, new ConcurrentHashSet<>());
    Set<JobConfig> childJobConfigs = workflowExecution.next();
    if (childJobConfigs.isEmpty()) {
        done(jobId);
        return;
    }
    ConcurrentHashSet<Long> childJobIds = new ConcurrentHashSet<>();
    for (int i = 0; i < childJobConfigs.size(); i++) {
        childJobIds.add(mJobMaster.getNewJobId());
    }
    mWaitingOn.put(jobId, childJobIds);
    mChildren.get(jobId).addAll(childJobIds);
    for (Long childJobId : childJobIds) {
        mParentWorkflow.put(childJobId, jobId);
    }
    Iterator<Long> childJobIdsIter = childJobIds.iterator();
    Iterator<JobConfig> childJobConfigsIter = childJobConfigs.iterator();
    while (childJobIdsIter.hasNext() && childJobConfigsIter.hasNext()) {
        Long childJobId = childJobIdsIter.next();
        JobConfig childJobConfig = childJobConfigsIter.next();
        try {
            mJobMaster.run(childJobConfig, childJobId);
        } catch (JobDoesNotExistException | ResourceExhaustedException e) {
            LOG.warn(e.getMessage());
            final String errorType = ErrorUtils.getErrorType(e);
            workflowExecution.stop(Status.FAILED, errorType, e.getMessage());
            stop(jobId, Status.FAILED, errorType, e.getMessage());
        }
    }
}