Examples with Lop org.apache.sysml.lops.Lop used on opensource projects

Example 41 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class Dag method handleSingleOutputJobs.

/**
 * As some jobs only write one output, all operations in the mapper need to
 * be redone and cannot be marked as finished.
 *
 * @param execNodes list of exec low-level operators
 * @param jobNodes list of job low-level operators
 * @param finishedNodes list of finished low-level operators
 */
private void handleSingleOutputJobs(ArrayList<Lop> execNodes, ArrayList<ArrayList<Lop>> jobNodes, ArrayList<Lop> finishedNodes) {
    /*
		 * If the input of a MMCJ/MMRJ job (must have executed in a Mapper) is used
		 * by multiple lops then we should mark it as not-finished.
		 */
    ArrayList<Lop> nodesWithUnfinishedOutputs = new ArrayList<>();
    int[] jobIndices = { JobType.MMCJ.getId() };
    Lop.Type[] lopTypes = { Lop.Type.MMCJ };
    for (int jobi = 0; jobi < jobIndices.length; jobi++) {
        int jindex = jobIndices[jobi];
        if (!jobNodes.get(jindex).isEmpty()) {
            ArrayList<Lop> vec = jobNodes.get(jindex);
            // first find all nodes with more than one parent that is not finished.
            for (int i = 0; i < vec.size(); i++) {
                Lop node = vec.get(i);
                if (node.getExecLocation() == ExecLocation.MapOrReduce || node.getExecLocation() == ExecLocation.Map) {
                    Lop MRparent = getParentNode(node, execNodes, ExecLocation.MapAndReduce);
                    if (MRparent != null && MRparent.getType() == lopTypes[jobi]) {
                        int numParents = node.getOutputs().size();
                        if (numParents > 1) {
                            for (int j = 0; j < numParents; j++) {
                                if (!finishedNodes.contains(node.getOutputs().get(j)))
                                    nodesWithUnfinishedOutputs.add(node);
                            }
                        }
                    }
                }
            }
            // need to redo all nodes in nodesWithOutput as well as their children
            for (Lop node : vec) {
                if (node.getExecLocation() == ExecLocation.MapOrReduce || node.getExecLocation() == ExecLocation.Map) {
                    if (nodesWithUnfinishedOutputs.contains(node))
                        finishedNodes.remove(node);
                    if (hasParentNode(node, nodesWithUnfinishedOutputs))
                        finishedNodes.remove(node);
                }
            }
        }
    }
}

Example 42 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class Dag method removeNodesForNextIteration.

/**
 * Method to remove all child nodes of a queued node that should be executed
 * in a following iteration.
 *
 * @param node low-level operator
 * @param finishedNodes list of finished nodes
 * @param execNodes list of exec nodes
 * @param queuedNodes list of queued nodes
 * @param jobvec list of lists of low-level operators
 */
private void removeNodesForNextIteration(Lop node, ArrayList<Lop> finishedNodes, ArrayList<Lop> execNodes, ArrayList<Lop> queuedNodes, ArrayList<ArrayList<Lop>> jobvec) {
    // only queued nodes with multiple inputs need to be handled.
    if (node.getInputs().size() == 1)
        return;
    // if all children are queued, then there is nothing to do.
    boolean allQueued = true;
    for (Lop input : node.getInputs()) {
        if (!queuedNodes.contains(input)) {
            allQueued = false;
            break;
        }
    }
    if (allQueued)
        return;
    if (LOG.isTraceEnabled())
        LOG.trace("  Before remove nodes for next iteration -- size of execNodes " + execNodes.size());
    // Determine if <code>node</code> has inputs from the same job or multiple jobs
    int jobid = Integer.MIN_VALUE;
    boolean inputs_in_same_job = true;
    for (Lop input : node.getInputs()) {
        int input_jobid = jobType(input, jobvec);
        if (jobid == Integer.MIN_VALUE)
            jobid = input_jobid;
        else if (jobid != input_jobid) {
            inputs_in_same_job = false;
            break;
        }
    }
    // Determine if there exist any unassigned inputs to <code>node</code>
    // Evaluate only those lops that execute in MR.
    boolean unassigned_inputs = false;
    for (Lop input : node.getInputs()) {
        if (input.getExecType() == ExecType.MR && !execNodes.contains(input)) {
            unassigned_inputs = true;
            break;
        }
    }
    // Determine if any node's children are queued
    boolean child_queued = false;
    for (Lop input : node.getInputs()) {
        if (queuedNodes.contains(input)) {
            child_queued = true;
            break;
        }
    }
    if (LOG.isTraceEnabled()) {
        LOG.trace("  Property Flags:");
        LOG.trace("    Inputs in same job: " + inputs_in_same_job);
        LOG.trace("    Unassigned inputs: " + unassigned_inputs);
        LOG.trace("    Child queued: " + child_queued);
    }
    // Evaluate each lop in <code>execNodes</code> for removal.
    // Add lops to be removed to <code>markedNodes</code>.
    ArrayList<Lop> markedNodes = new ArrayList<>();
    for (Lop tmpNode : execNodes) {
        if (LOG.isTraceEnabled()) {
            LOG.trace("  Checking for removal (" + tmpNode.getID() + ") " + tmpNode.toString());
        }
        // if tmpNode is not a descendant of 'node', then there is no advantage in removing tmpNode for later iterations.
        if (!isChild(tmpNode, node, IDMap))
            continue;
        // handle group input lops
        if (node.getInputs().contains(tmpNode) && tmpNode.isAligner()) {
            markedNodes.add(tmpNode);
            if (LOG.isTraceEnabled())
                LOG.trace("    Removing for next iteration (code 1): (" + tmpNode.getID() + ") " + tmpNode.toString());
        }
        if (!hasOtherQueuedParentNode(tmpNode, queuedNodes, node) && branchHasNoOtherUnExecutedParents(tmpNode, node, execNodes, finishedNodes)) {
            boolean queueit = false;
            int code = -1;
            switch(node.getExecLocation()) {
                case Map:
                    if (branchCanBePiggyBackedMap(tmpNode, node, execNodes, queuedNodes, markedNodes))
                        queueit = true;
                    code = 2;
                    break;
                case MapAndReduce:
                    if (branchCanBePiggyBackedMapAndReduce(tmpNode, node, execNodes, queuedNodes) && !tmpNode.definesMRJob())
                        queueit = true;
                    code = 3;
                    break;
                case Reduce:
                    if (branchCanBePiggyBackedReduce(tmpNode, node, execNodes, queuedNodes))
                        queueit = true;
                    code = 4;
                    break;
                default:
            }
            if (queueit) {
                if (LOG.isTraceEnabled())
                    LOG.trace("    Removing for next iteration (code " + code + "): (" + tmpNode.getID() + ") " + tmpNode.toString());
                markedNodes.add(tmpNode);
            }
        }
        /*
				 * "node" has no other queued children.
				 * 
				 * If inputs are in the same job and "node" is of type
				 * MapAndReduce, then remove nodes of all types other than
				 * Reduce, MapAndReduce, and the ones that define a MR job as
				 * they can be piggybacked later.
				 * 
				 * e.g: A=Rand, B=Rand, C=A%*%B Here, both inputs of MMCJ lop
				 * come from Rand job, and they should not be removed.
				 * 
				 * Other examples: -- MMCJ whose children are of type
				 * MapAndReduce (say GMR) -- Inputs coming from two different
				 * jobs .. GMR & REBLOCK
				 */
        if ((inputs_in_same_job || unassigned_inputs) && node.getExecLocation() == ExecLocation.MapAndReduce && // don't remove since it already piggybacked with a MapReduce node
        !hasOtherMapAndReduceParentNode(tmpNode, execNodes, node) && branchCanBePiggyBackedMapAndReduce(tmpNode, node, execNodes, queuedNodes) && !tmpNode.definesMRJob()) {
            if (LOG.isTraceEnabled())
                LOG.trace("    Removing for next iteration (code 5): (" + tmpNode.getID() + ") " + tmpNode.toString());
            markedNodes.add(tmpNode);
        }
    }
    // we also need to delete all parent nodes of marked nodes
    for (Lop enode : execNodes) {
        if (LOG.isTraceEnabled()) {
            LOG.trace("  Checking for removal - (" + enode.getID() + ") " + enode.toString());
        }
        if (hasChildNode(enode, markedNodes) && !markedNodes.contains(enode)) {
            markedNodes.add(enode);
            if (LOG.isTraceEnabled())
                LOG.trace("    Removing for next iteration (code 6) (" + enode.getID() + ") " + enode.toString());
        }
    }
    if (execNodes.size() != markedNodes.size()) {
        // add to queued nodes
        for (Lop n : markedNodes) {
            if (n.usesDistributedCache())
                gmrMapperFootprint -= computeFootprintInMapper(n);
            finishedNodes.remove(n);
            execNodes.remove(n);
            removeNodeByJobType(n, jobvec);
            queuedNodes.add(n);
        }
    }
}

Example 43 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class Dag method sendWriteLopToMR.

/**
 * Determine whether to send <code>node</code> to MR or to process it in the control program.
 * It is sent to MR in the following cases:
 *
 * 1) if input lop gets processed in MR then <code>node</code> can be piggybacked
 *
 * 2) if the exectype of write lop itself is marked MR i.e., memory estimate > memory budget.
 *
 * @param node low-level operator
 * @return true if lop should be sent to MR
 */
private static boolean sendWriteLopToMR(Lop node) {
    if (DMLScript.rtplatform == RUNTIME_PLATFORM.SINGLE_NODE)
        return false;
    Lop in = node.getInputs().get(0);
    Format nodeFormat = node.getOutputParameters().getFormat();
    // not apply to csv write because MR csvwrite is a separate MR job type)
    return (node.getExecType() == ExecType.MR || (in.getExecType() == ExecType.MR && nodeFormat != Format.CSV));
}

Example 44 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class Dag method dagDFS.

/**
 * Method to perform depth-first traversal from a given node in the DAG.
 * Store the reachability information in marked[] boolean array.
 *
 * @param root low-level operator
 * @param marked reachability results
 */
private void dagDFS(Lop root, boolean[] marked) {
    // contains check currently required for globalopt, will be removed when cleaned up
    if (!IDMap.containsKey(root.getID()))
        return;
    int mapID = IDMap.get(root.getID());
    if (marked[mapID])
        return;
    marked[mapID] = true;
    for (Lop lop : root.getOutputs()) {
        dagDFS(lop, marked);
    }
}

Example 45 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class DMLTranslator method constructLops.

public boolean constructLops(StatementBlock sb) {
    boolean ret = false;
    if (sb instanceof WhileStatementBlock) {
        WhileStatementBlock wsb = (WhileStatementBlock) sb;
        WhileStatement whileStmt = (WhileStatement) wsb.getStatement(0);
        ArrayList<StatementBlock> body = whileStmt.getBody();
        // step through stmt blocks in while stmt body
        for (StatementBlock stmtBlock : body) ret |= constructLops(stmtBlock);
        // handle while stmt predicate
        Lop l = wsb.getPredicateHops().constructLops();
        wsb.set_predicateLops(l);
        ret |= wsb.updatePredicateRecompilationFlag();
    } else if (sb instanceof IfStatementBlock) {
        IfStatementBlock isb = (IfStatementBlock) sb;
        IfStatement ifStmt = (IfStatement) isb.getStatement(0);
        ArrayList<StatementBlock> ifBody = ifStmt.getIfBody();
        ArrayList<StatementBlock> elseBody = ifStmt.getElseBody();
        // step through stmt blocks in if stmt ifBody
        for (StatementBlock stmtBlock : ifBody) ret |= constructLops(stmtBlock);
        // step through stmt blocks in if stmt elseBody
        for (StatementBlock stmtBlock : elseBody) ret |= constructLops(stmtBlock);
        // handle if stmt predicate
        Lop l = isb.getPredicateHops().constructLops();
        isb.set_predicateLops(l);
        ret |= isb.updatePredicateRecompilationFlag();
    } else if (// NOTE: applies to ForStatementBlock and ParForStatementBlock
    sb instanceof ForStatementBlock) {
        ForStatementBlock fsb = (ForStatementBlock) sb;
        ForStatement fs = (ForStatement) sb.getStatement(0);
        ArrayList<StatementBlock> body = fs.getBody();
        // step through stmt blocks in FOR stmt body
        for (StatementBlock stmtBlock : body) ret |= constructLops(stmtBlock);
        // handle for stmt predicate
        if (fsb.getFromHops() != null) {
            Lop llobs = fsb.getFromHops().constructLops();
            fsb.setFromLops(llobs);
        }
        if (fsb.getToHops() != null) {
            Lop llobs = fsb.getToHops().constructLops();
            fsb.setToLops(llobs);
        }
        if (fsb.getIncrementHops() != null) {
            Lop llobs = fsb.getIncrementHops().constructLops();
            fsb.setIncrementLops(llobs);
        }
        ret |= fsb.updatePredicateRecompilationFlags();
    } else if (sb instanceof FunctionStatementBlock) {
        FunctionStatementBlock fsb = (FunctionStatementBlock) sb;
        FunctionStatement functStmt = (FunctionStatement) sb.getStatement(0);
        ArrayList<StatementBlock> body = functStmt.getBody();
        // step through stmt blocks in while stmt body
        for (StatementBlock stmtBlock : body) ret |= constructLops(stmtBlock);
        if (fsb.isRecompileOnce())
            fsb.setRecompileOnce(ret);
    } else // handle default case for regular StatementBlock
    {
        if (sb.getHops() == null)
            sb.setHops(new ArrayList<Hop>());
        ArrayList<Lop> lops = new ArrayList<>();
        for (Hop hop : sb.getHops()) lops.add(hop.constructLops());
        sb.setLops(lops);
        ret |= sb.updateRecompilationFlag();
    }
    return ret;
}