Examples with DeferredOp com.tdunning.plume.local.lazy.op.DeferredOp used on opensource projects

Example 11 with DeferredOp

use of com.tdunning.plume.local.lazy.op.DeferredOp in project Plume by tdunning.

the class Optimizer method fuseSiblingParallelDos.

/**
 * Join ParallelDos that use the same PCollection into multiple-output {@link MultipleParallelDo}
 * @param arg  The original collection that may contain sibling do chains
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
<T> void fuseSiblingParallelDos(PCollection<T> arg) {
    LazyCollection<T> output = (LazyCollection<T>) arg;
    if (output.isMaterialized()) {
        // stop condition for recursive algorithm
        return;
    }
    DeferredOp dOp = output.getDeferredOp();
    if (!(dOp instanceof ParallelDo)) {
        // not a ParallelDo
        if (dOp instanceof OneToOneOp) {
            // Recursively apply this function to parent
            fuseSiblingParallelDos(((OneToOneOp) dOp).getOrigin());
            return;
        }
        if (dOp instanceof Flatten) {
            Flatten<T> flatten = (Flatten) dOp;
            // Recursively apply this function to all parents
            for (PCollection<T> col : flatten.getOrigins()) {
                fuseSiblingParallelDos(col);
            }
            return;
        }
        if (dOp instanceof MultipleParallelDo) {
            return;
        }
    }
    ParallelDo pDo = (ParallelDo) output.getDeferredOp();
    LazyCollection<T> orig = (LazyCollection<T>) pDo.getOrigin();
    int willAdd = 0;
    for (DeferredOp op : orig.getDownOps()) {
        if (op instanceof ParallelDo) {
            willAdd++;
        }
    }
    if (willAdd == 1) {
        // Parent doesn't have more ParallelDos to fuse
        // Recursively apply this function to parent
        fuseSiblingParallelDos(orig);
        return;
    }
    // MultipleParallelDo is viable, create it
    MultipleParallelDo<T> mPDo = new MultipleParallelDo<T>(orig);
    mPDo.addDest(pDo.getFunction(), output);
    orig.downOps.remove(pDo);
    output.deferredOp = mPDo;
    List<DeferredOp> newList = new ArrayList<DeferredOp>();
    for (DeferredOp op : orig.getDownOps()) {
        if (op instanceof ParallelDo) {
            ParallelDo thisPDo = (ParallelDo) op;
            mPDo.addDest(thisPDo.getFunction(), thisPDo.getDest());
            LazyCollection thisDest = (LazyCollection) thisPDo.getDest();
            thisDest.deferredOp = mPDo;
        } else {
            newList.add(op);
        }
    }
    newList.add(mPDo);
    orig.downOps = newList;
    // Recursively apply this function to parent
    fuseSiblingParallelDos(orig);
}