Examples with CrunchControlledJob org.apache.hadoop.mapreduce.lib.jobcontrol.CrunchControlledJob used on opensource projects

Example 1 with CrunchControlledJob

use of org.apache.hadoop.mapreduce.lib.jobcontrol.CrunchControlledJob in project crunch by cloudera.

the class MRExecutor method execute.

public PipelineResult execute() {
    try {
        Thread controlThread = new Thread(control);
        controlThread.start();
        while (!control.allFinished()) {
            Thread.sleep(1000);
        }
        control.stop();
    } catch (InterruptedException e) {
        LOG.info(e);
    }
    List<CrunchControlledJob> failures = control.getFailedJobList();
    if (!failures.isEmpty()) {
        System.err.println(failures.size() + " job failure(s) occurred:");
        for (CrunchControlledJob job : failures) {
            System.err.println(job.getJobName() + "(" + job.getJobID() + "): " + job.getMessage());
        }
    }
    List<PipelineResult.StageResult> stages = Lists.newArrayList();
    for (CrunchControlledJob job : control.getSuccessfulJobList()) {
        try {
            stages.add(new PipelineResult.StageResult(job.getJobName(), job.getJob().getCounters()));
        } catch (Exception e) {
            LOG.error("Exception thrown fetching job counters for stage: " + job.getJobName(), e);
        }
    }
    return new PipelineResult(stages);
}