Examples with MaterializableIterable org.apache.crunch.materialize.MaterializableIterable used on opensource projects

Example 1 with MaterializableIterable

use of org.apache.crunch.materialize.MaterializableIterable in project crunch by cloudera.

the class MRPipeline method run.

@Override
public PipelineResult run() {
    MSCRPlanner planner = new MSCRPlanner(this, outputTargets);
    PipelineResult res = null;
    try {
        res = planner.plan(jarClass, conf).execute();
    } catch (IOException e) {
        LOG.error(e);
        return PipelineResult.EMPTY;
    }
    for (PCollectionImpl<?> c : outputTargets.keySet()) {
        if (outputTargetsToMaterialize.containsKey(c)) {
            MaterializableIterable iter = outputTargetsToMaterialize.get(c);
            iter.materialize();
            c.materializeAt(iter.getSourceTarget());
            outputTargetsToMaterialize.remove(c);
        } else {
            boolean materialized = false;
            for (Target t : outputTargets.get(c)) {
                if (!materialized && t instanceof Source) {
                    c.materializeAt((SourceTarget) t);
                    materialized = true;
                }
            }
        }
    }
    outputTargets.clear();
    return res;
}