Examples with PipelineResult org.apache.crunch.PipelineResult used on opensource projects

Example 1 with PipelineResult

use of org.apache.crunch.PipelineResult in project crunch by cloudera.

the class MRPipeline method done.

@Override
public PipelineResult done() {
    PipelineResult res = null;
    if (!outputTargets.isEmpty()) {
        res = run();
    }
    cleanup();
    return res;
}

Example 2 with PipelineResult

use of org.apache.crunch.PipelineResult in project crunch by cloudera.

the class MRExecutor method execute.

public PipelineResult execute() {
    try {
        Thread controlThread = new Thread(control);
        controlThread.start();
        while (!control.allFinished()) {
            Thread.sleep(1000);
        }
        control.stop();
    } catch (InterruptedException e) {
        LOG.info(e);
    }
    List<CrunchControlledJob> failures = control.getFailedJobList();
    if (!failures.isEmpty()) {
        System.err.println(failures.size() + " job failure(s) occurred:");
        for (CrunchControlledJob job : failures) {
            System.err.println(job.getJobName() + "(" + job.getJobID() + "): " + job.getMessage());
        }
    }
    List<PipelineResult.StageResult> stages = Lists.newArrayList();
    for (CrunchControlledJob job : control.getSuccessfulJobList()) {
        try {
            stages.add(new PipelineResult.StageResult(job.getJobName(), job.getJob().getCounters()));
        } catch (Exception e) {
            LOG.error("Exception thrown fetching job counters for stage: " + job.getJobName(), e);
        }
    }
    return new PipelineResult(stages);
}

Example 3 with PipelineResult

use of org.apache.crunch.PipelineResult in project crunch by cloudera.

the class MRPipeline method run.

@Override
public PipelineResult run() {
    MSCRPlanner planner = new MSCRPlanner(this, outputTargets);
    PipelineResult res = null;
    try {
        res = planner.plan(jarClass, conf).execute();
    } catch (IOException e) {
        LOG.error(e);
        return PipelineResult.EMPTY;
    }
    for (PCollectionImpl<?> c : outputTargets.keySet()) {
        if (outputTargetsToMaterialize.containsKey(c)) {
            MaterializableIterable iter = outputTargetsToMaterialize.get(c);
            iter.materialize();
            c.materializeAt(iter.getSourceTarget());
            outputTargetsToMaterialize.remove(c);
        } else {
            boolean materialized = false;
            for (Target t : outputTargets.get(c)) {
                if (!materialized && t instanceof Source) {
                    c.materializeAt((SourceTarget) t);
                    materialized = true;
                }
            }
        }
    }
    outputTargets.clear();
    return res;
}