Examples with Lop org.apache.sysml.lops.Lop used on opensource projects

Example 96 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class Dag method getRecordReaderInstructions.

/**
 * Method to get record reader instructions for a MR job.
 *
 * @param node low-level operator
 * @param execNodes list of exec nodes
 * @param inputStrings list of input strings
 * @param recordReaderInstructions list of record reader instructions
 * @param nodeIndexMapping node index mapping
 * @param start_index start index
 * @param inputLabels list of input labels
 * @param inputLops list of input lops
 * @param MRJobLineNumbers MR job line numbers
 * @return -1 if problem
 */
private static int getRecordReaderInstructions(Lop node, ArrayList<Lop> execNodes, ArrayList<String> inputStrings, ArrayList<String> recordReaderInstructions, HashMap<Lop, Integer> nodeIndexMapping, int[] start_index, ArrayList<String> inputLabels, ArrayList<Lop> inputLops, ArrayList<Integer> MRJobLineNumbers) {
    // if input source, return index
    if (nodeIndexMapping.containsKey(node))
        return nodeIndexMapping.get(node);
    // not input source and not in exec nodes, then return.
    if (!execNodes.contains(node))
        return -1;
    ArrayList<Integer> inputIndices = new ArrayList<>();
    int max_input_index = -1;
    // get mapper instructions
    for (int i = 0; i < node.getInputs().size(); i++) {
        // recurse
        Lop childNode = node.getInputs().get(i);
        int ret_val = getRecordReaderInstructions(childNode, execNodes, inputStrings, recordReaderInstructions, nodeIndexMapping, start_index, inputLabels, inputLops, MRJobLineNumbers);
        inputIndices.add(ret_val);
        if (ret_val > max_input_index) {
            max_input_index = ret_val;
        // child_for_max_input_index = childNode;
        }
    }
    // instructions
    if ((node.getExecLocation() == ExecLocation.RecordReader)) {
        int output_index = max_input_index;
        // cannot reuse index if this is true
        // need to add better indexing schemes
        output_index = start_index[0];
        start_index[0]++;
        nodeIndexMapping.put(node, output_index);
        // only Ranagepick lop can contribute to labels
        if (node.getType() == Type.PickValues) {
            PickByCount pbc = (PickByCount) node;
            if (pbc.getOperationType() == PickByCount.OperationTypes.RANGEPICK) {
                // always the second input is a scalar
                int scalarIndex = 1;
                // if data lop not a literal -- add label
                if (node.getInputs().get(scalarIndex).getExecLocation() == ExecLocation.Data && !((Data) (node.getInputs().get(scalarIndex))).isLiteral()) {
                    inputLabels.add(node.getInputs().get(scalarIndex).getOutputParameters().getLabel());
                    inputLops.add(node.getInputs().get(scalarIndex));
                }
                // if not data lop, then this is an intermediate variable.
                if (node.getInputs().get(scalarIndex).getExecLocation() != ExecLocation.Data) {
                    inputLabels.add(node.getInputs().get(scalarIndex).getOutputParameters().getLabel());
                    inputLops.add(node.getInputs().get(scalarIndex));
                }
            }
        }
        // get recordreader instruction.
        if (node.getInputs().size() == 2) {
            recordReaderInstructions.add(node.getInstructions(inputIndices.get(0), inputIndices.get(1), output_index));
            if (DMLScript.ENABLE_DEBUG_MODE) {
                MRJobLineNumbers.add(node._beginLine);
            }
        } else
            throw new LopsException("Unexpected number of inputs while generating a RecordReader Instruction");
        return output_index;
    }
    return -1;
}

Example 97 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class Dag method generateInstructionsForInputVariables.

/**
 * Method to generate createvar instructions, which creates a new entry
 * in the symbol table. One instruction is generated for every LOP that is
 * 1) type Data and
 * 2) persistent and
 * 3) matrix and
 * 4) read
 *
 * Transient reads needn't be considered here since the previous program
 * block would already create appropriate entries in the symbol table.
 *
 * @param nodes_v list of nodes
 * @param inst list of instructions
 */
private static void generateInstructionsForInputVariables(ArrayList<Lop> nodes_v, ArrayList<Instruction> inst) {
    for (Lop n : nodes_v) {
        if (n.getExecLocation() == ExecLocation.Data && !((Data) n).isTransient() && ((Data) n).getOperationType() == OperationTypes.READ && (n.getDataType() == DataType.MATRIX || n.getDataType() == DataType.FRAME)) {
            if (!((Data) n).isLiteral()) {
                try {
                    String inst_string = n.getInstructions();
                    CPInstruction currInstr = CPInstructionParser.parseSingleInstruction(inst_string);
                    currInstr.setLocation(n);
                    inst.add(currInstr);
                } catch (DMLRuntimeException e) {
                    throw new LopsException(n.printErrorLocation() + "error generating instructions from input variables in Dag -- \n", e);
                }
            }
        }
    }
}

Example 98 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class Dag method deleteUpdatedTransientReadVariables.

private static void deleteUpdatedTransientReadVariables(StatementBlock sb, ArrayList<Lop> nodeV, ArrayList<Instruction> inst) {
    if (sb == null)
        return;
    if (LOG.isTraceEnabled())
        LOG.trace("In delete updated variables");
    // CANDIDATE list of variables which could have been updated in this statement block
    HashMap<String, Lop> labelNodeMapping = new HashMap<>();
    // ACTUAL list of variables whose value is updated, AND the old value of the variable
    // is no longer accessible/used.
    HashSet<String> updatedLabels = new HashSet<>();
    HashMap<String, Lop> updatedLabelsLineNum = new HashMap<>();
    // first capture all transient read variables
    for (Lop node : nodeV) {
        if (node.getExecLocation() == ExecLocation.Data && ((Data) node).isTransient() && ((Data) node).getOperationType() == OperationTypes.READ && ((Data) node).getDataType() == DataType.MATRIX) {
            // "node" is considered as updated ONLY IF the old value is not used any more
            // So, make sure that this READ node does not feed into any (transient/persistent) WRITE
            boolean hasWriteParent = false;
            for (Lop p : node.getOutputs()) {
                if (p.getExecLocation() == ExecLocation.Data) {
                    // if the "p" is of type Data, then it has to be a WRITE
                    hasWriteParent = true;
                    break;
                }
            }
            if (!hasWriteParent) {
                // node has no parent of type WRITE, so this is a CANDIDATE variable
                // add it to labelNodeMapping so that it is considered in further processing
                labelNodeMapping.put(node.getOutputParameters().getLabel(), node);
            }
        }
    }
    // capture updated transient write variables
    for (Lop node : nodeV) {
        if (node.getExecLocation() == ExecLocation.Data && ((Data) node).isTransient() && ((Data) node).getOperationType() == OperationTypes.WRITE && ((Data) node).getDataType() == DataType.MATRIX && // check to make sure corresponding (i.e., with the same label/name) transient read is present
        labelNodeMapping.containsKey(node.getOutputParameters().getLabel()) && // check to avoid cases where transient read feeds into a transient write
        !labelNodeMapping.containsValue(node.getInputs().get(0))) {
            updatedLabels.add(node.getOutputParameters().getLabel());
            updatedLabelsLineNum.put(node.getOutputParameters().getLabel(), node);
        }
    }
    // generate RM instructions
    Instruction rm_inst = null;
    for (String label : updatedLabels) {
        rm_inst = VariableCPInstruction.prepareRemoveInstruction(label);
        rm_inst.setLocation(updatedLabelsLineNum.get(label));
        if (LOG.isTraceEnabled())
            LOG.trace(rm_inst.toString());
        inst.add(rm_inst);
    }
}

Example 99 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class Dag method doGreedyGrouping.

/**
 * Method to group a vector of sorted lops.
 *
 * @param sb statement block
 * @param node_v list of low-level operators
 * @return list of instructions
 */
private ArrayList<Instruction> doGreedyGrouping(StatementBlock sb, ArrayList<Lop> node_v) {
    if (LOG.isTraceEnabled())
        LOG.trace("Grouping DAG ============");
    // nodes to be executed in current iteration
    ArrayList<Lop> execNodes = new ArrayList<>();
    // nodes that have already been processed
    ArrayList<Lop> finishedNodes = new ArrayList<>();
    // nodes that are queued for the following iteration
    ArrayList<Lop> queuedNodes = new ArrayList<>();
    ArrayList<ArrayList<Lop>> jobNodes = createNodeVectors(JobType.getNumJobTypes());
    // list of instructions
    ArrayList<Instruction> inst = new ArrayList<>();
    // ArrayList<Instruction> preWriteDeleteInst = new ArrayList<Instruction>();
    ArrayList<Instruction> writeInst = new ArrayList<>();
    ArrayList<Instruction> deleteInst = new ArrayList<>();
    ArrayList<Instruction> endOfBlockInst = new ArrayList<>();
    // remove files for transient reads that are updated.
    deleteUpdatedTransientReadVariables(sb, node_v, writeInst);
    generateRemoveInstructions(sb, endOfBlockInst);
    generateInstructionsForInputVariables(node_v, inst);
    boolean done = false;
    String indent = "    ";
    while (!done) {
        if (LOG.isTraceEnabled())
            LOG.trace("Grouping nodes in DAG");
        execNodes.clear();
        queuedNodes.clear();
        clearNodeVectors(jobNodes);
        gmrMapperFootprint = 0;
        for (Lop node : node_v) {
            // finished nodes don't need to be processed
            if (finishedNodes.contains(node))
                continue;
            if (LOG.isTraceEnabled())
                LOG.trace("Processing node (" + node.getID() + ") " + node.toString() + " exec nodes size is " + execNodes.size());
            // its children nodes in execNodes
            if (node.definesMRJob() && !compatibleWithChildrenInExecNodes(execNodes, node)) {
                if (LOG.isTraceEnabled())
                    LOG.trace(indent + "Queueing node " + node.toString() + " (code 1)");
                queuedNodes.add(node);
                removeNodesForNextIteration(node, finishedNodes, execNodes, queuedNodes, jobNodes);
                continue;
            }
            // iteration
            if (hasChildNode(node, queuedNodes)) {
                if (LOG.isTraceEnabled())
                    LOG.trace(indent + "Queueing node " + node.toString() + " (code 2)");
                queuedNodes.add(node);
                // if node has more than two inputs,
                // remove children that will be needed in a future
                // iterations
                // may also have to remove parent nodes of these children
                removeNodesForNextIteration(node, finishedNodes, execNodes, queuedNodes, jobNodes);
                continue;
            }
            // if inputs come from different jobs, then queue
            if (node.getInputs().size() >= 2) {
                int jobid = Integer.MIN_VALUE;
                boolean queueit = false;
                for (int idx = 0; idx < node.getInputs().size(); idx++) {
                    int input_jobid = jobType(node.getInputs().get(idx), jobNodes);
                    if (input_jobid != -1) {
                        if (jobid == Integer.MIN_VALUE)
                            jobid = input_jobid;
                        else if (jobid != input_jobid) {
                            queueit = true;
                            break;
                        }
                    }
                }
                if (queueit) {
                    if (LOG.isTraceEnabled())
                        LOG.trace(indent + "Queueing node " + node.toString() + " (code 3)");
                    queuedNodes.add(node);
                    removeNodesForNextIteration(node, finishedNodes, execNodes, queuedNodes, jobNodes);
                    continue;
                }
            }
            // See if this lop can be eliminated
            // This check is for "aligner" lops (e.g., group)
            boolean eliminate = false;
            eliminate = canEliminateLop(node, execNodes);
            if (eliminate) {
                if (LOG.isTraceEnabled())
                    LOG.trace(indent + "Adding -" + node.toString());
                execNodes.add(node);
                finishedNodes.add(node);
                addNodeByJobType(node, jobNodes, execNodes, eliminate);
                continue;
            }
            // children that defines a MR Job are present in execNodes
            if (node.definesMRJob()) {
                if (hasMRJobChildNode(node, execNodes)) {
                    // this is because "group" can be pushed into the "Rand" job.
                    if (!(node.getType() == Lop.Type.Grouping && checkDataGenAsChildNode(node, execNodes))) {
                        if (LOG.isTraceEnabled())
                            LOG.trace(indent + "Queueing node " + node.toString() + " (code 4)");
                        queuedNodes.add(node);
                        removeNodesForNextIteration(node, finishedNodes, execNodes, queuedNodes, jobNodes);
                        continue;
                    }
                }
            }
            // not, queue "node"
            if (node.getInputs().size() > 1 && hasChildNode(node, execNodes, ExecLocation.RecordReader)) {
                // get the actual RecordReader lop
                Lop rr_node = getChildNode(node, execNodes, ExecLocation.RecordReader);
                // all inputs of "node" must be ancestors of rr_node
                boolean queue_it = false;
                for (Lop n : node.getInputs()) {
                    // each input should be ancestor of RecordReader lop
                    if (!n.equals(rr_node) && !isChild(rr_node, n, IDMap)) {
                        // i.e., "node" must be queued
                        queue_it = true;
                        break;
                    }
                }
                if (queue_it) {
                    // queue node
                    if (LOG.isTraceEnabled())
                        LOG.trace(indent + "Queueing -" + node.toString() + " (code 5)");
                    queuedNodes.add(node);
                    // TODO: does this have to be modified to handle
                    // recordreader lops?
                    removeNodesForNextIteration(node, finishedNodes, execNodes, queuedNodes, jobNodes);
                    continue;
                } else {
                // nothing here.. subsequent checks have to be performed
                // on "node"
                }
            }
            // only write nodes are kept in execnodes
            if (node.getExecLocation() == ExecLocation.Data) {
                Data dnode = (Data) node;
                boolean dnode_queued = false;
                if (dnode.getOperationType() == OperationTypes.READ) {
                    if (LOG.isTraceEnabled())
                        LOG.trace(indent + "Adding Data -" + node.toString());
                    // TODO: avoid readScalar instruction, and read it on-demand just like the way Matrices are read in control program
                    if (node.getDataType() == DataType.SCALAR && // TODO: LEO check the following condition is still needed
                    node.getOutputParameters().getFile_name() != null) {
                        // this lop corresponds to reading a scalar from HDFS file
                        // add it to execNodes so that "readScalar" instruction gets generated
                        execNodes.add(node);
                    // note: no need to add it to any job vector
                    }
                } else if (dnode.getOperationType() == OperationTypes.WRITE) {
                    // Skip the transient write <code>node</code> if the input is a
                    // transient read with the same variable name. i.e., a dummy copy.
                    // Hence, <code>node</code> can be avoided.
                    // TODO: this case should ideally be handled in the language layer
                    // prior to the construction of Hops Dag
                    Lop input = dnode.getInputs().get(0);
                    if (dnode.isTransient() && input.getExecLocation() == ExecLocation.Data && ((Data) input).isTransient() && dnode.getOutputParameters().getLabel().equals(input.getOutputParameters().getLabel())) {
                    // do nothing, <code>node</code> must not processed any further.
                    } else if (execNodes.contains(input) && !isCompatible(node, input) && sendWriteLopToMR(node)) {
                        // input is in execNodes but it is not compatible with write lop. So, queue the write lop.
                        if (LOG.isTraceEnabled())
                            LOG.trace(indent + "Queueing -" + node.toString());
                        queuedNodes.add(node);
                        dnode_queued = true;
                    } else {
                        if (LOG.isTraceEnabled())
                            LOG.trace(indent + "Adding Data -" + node.toString());
                        execNodes.add(node);
                        if (sendWriteLopToMR(node)) {
                            addNodeByJobType(node, jobNodes, execNodes, false);
                        }
                    }
                }
                if (!dnode_queued)
                    finishedNodes.add(node);
                continue;
            }
            // map or reduce node, can always be piggybacked with parent
            if (node.getExecLocation() == ExecLocation.MapOrReduce) {
                if (LOG.isTraceEnabled())
                    LOG.trace(indent + "Adding -" + node.toString());
                execNodes.add(node);
                finishedNodes.add(node);
                addNodeByJobType(node, jobNodes, execNodes, false);
                continue;
            }
            // RecordReader node, add, if no parent needs reduce, else queue
            if (node.getExecLocation() == ExecLocation.RecordReader) {
                // execNodes .. it has to be the first one in the job!
                if (!hasChildNode(node, execNodes, ExecLocation.Map) && !hasChildNode(node, execNodes, ExecLocation.MapAndReduce)) {
                    if (LOG.isTraceEnabled())
                        LOG.trace(indent + "Adding -" + node.toString());
                    execNodes.add(node);
                    finishedNodes.add(node);
                    addNodeByJobType(node, jobNodes, execNodes, false);
                } else {
                    if (LOG.isTraceEnabled())
                        LOG.trace(indent + "Queueing -" + node.toString() + " (code 6)");
                    queuedNodes.add(node);
                    removeNodesForNextIteration(node, finishedNodes, execNodes, queuedNodes, jobNodes);
                }
                continue;
            }
            // map node, add, if no parent needs reduce, else queue
            if (node.getExecLocation() == ExecLocation.Map) {
                boolean queueThisNode = false;
                int subcode = -1;
                if (node.usesDistributedCache()) {
                    // if an input to <code>node</code> comes from distributed cache
                    // then that input must get executed in one of the previous jobs.
                    int[] dcInputIndexes = node.distributedCacheInputIndex();
                    for (int dcInputIndex : dcInputIndexes) {
                        Lop dcInput = node.getInputs().get(dcInputIndex - 1);
                        if ((dcInput.getType() != Lop.Type.Data && dcInput.getExecType() == ExecType.MR) && execNodes.contains(dcInput)) {
                            queueThisNode = true;
                            subcode = 1;
                        }
                    }
                    // Limit the number of distributed cache inputs based on the available memory in mappers
                    double memsize = computeFootprintInMapper(node);
                    if (gmrMapperFootprint > 0 && !checkMemoryLimits(node, gmrMapperFootprint + memsize)) {
                        queueThisNode = true;
                        subcode = 2;
                    }
                    if (!queueThisNode)
                        gmrMapperFootprint += memsize;
                }
                if (!queueThisNode && !hasChildNode(node, execNodes, ExecLocation.MapAndReduce) && !hasMRJobChildNode(node, execNodes)) {
                    if (LOG.isTraceEnabled())
                        LOG.trace(indent + "Adding -" + node.toString());
                    execNodes.add(node);
                    finishedNodes.add(node);
                    addNodeByJobType(node, jobNodes, execNodes, false);
                } else {
                    if (LOG.isTraceEnabled())
                        LOG.trace(indent + "Queueing -" + node.toString() + " (code 7 - " + "subcode " + subcode + ")");
                    queuedNodes.add(node);
                    removeNodesForNextIteration(node, finishedNodes, execNodes, queuedNodes, jobNodes);
                }
                continue;
            }
            // reduce node, make sure no parent needs reduce, else queue
            if (node.getExecLocation() == ExecLocation.MapAndReduce) {
                // not define a job
                if (LOG.isTraceEnabled())
                    LOG.trace(indent + "Adding -" + node.toString());
                execNodes.add(node);
                finishedNodes.add(node);
                addNodeByJobType(node, jobNodes, execNodes, eliminate);
                continue;
            }
            // aligned reduce, make sure a parent that is reduce exists
            if (node.getExecLocation() == ExecLocation.Reduce) {
                if (compatibleWithChildrenInExecNodes(execNodes, node) && (hasChildNode(node, execNodes, ExecLocation.MapAndReduce) || hasChildNode(node, execNodes, ExecLocation.Map))) {
                    if (LOG.isTraceEnabled())
                        LOG.trace(indent + "Adding -" + node.toString());
                    execNodes.add(node);
                    finishedNodes.add(node);
                    addNodeByJobType(node, jobNodes, execNodes, false);
                } else {
                    if (LOG.isTraceEnabled())
                        LOG.trace(indent + "Queueing -" + node.toString() + " (code 8)");
                    queuedNodes.add(node);
                    removeNodesForNextIteration(node, finishedNodes, execNodes, queuedNodes, jobNodes);
                }
                continue;
            }
            // that will be executed in a MR job.
            if (node.getExecLocation() == ExecLocation.ControlProgram) {
                for (Lop lop : node.getInputs()) {
                    if (execNodes.contains(lop) && !(lop.getExecLocation() == ExecLocation.Data) && !(lop.getExecLocation() == ExecLocation.ControlProgram)) {
                        if (LOG.isTraceEnabled())
                            LOG.trace(indent + "Queueing -" + node.toString() + " (code 9)");
                        queuedNodes.add(node);
                        removeNodesForNextIteration(node, finishedNodes, execNodes, queuedNodes, jobNodes);
                        break;
                    }
                }
                if (queuedNodes.contains(node))
                    continue;
                if (LOG.isTraceEnabled())
                    LOG.trace(indent + "Adding - scalar" + node.toString());
                execNodes.add(node);
                addNodeByJobType(node, jobNodes, execNodes, false);
                finishedNodes.add(node);
                continue;
            }
        }
        // no work to do
        if (execNodes.isEmpty()) {
            if (!queuedNodes.isEmpty())
                throw new LopsException("Queued nodes should not be 0 at this point \n");
            if (LOG.isTraceEnabled())
                LOG.trace("All done! queuedNodes = " + queuedNodes.size());
            done = true;
        } else {
            if (LOG.isTraceEnabled())
                LOG.trace("Generating jobs for group -- Node count=" + execNodes.size());
            // first process scalar instructions
            generateControlProgramJobs(execNodes, inst, writeInst, deleteInst);
            // copy unassigned lops in execnodes to gmrnodes
            for (int i = 0; i < execNodes.size(); i++) {
                Lop node = execNodes.get(i);
                if (jobType(node, jobNodes) == -1) {
                    if (isCompatible(node, JobType.GMR)) {
                        if (node.hasNonBlockedInputs()) {
                            jobNodes.get(JobType.GMRCELL.getId()).add(node);
                            addChildren(node, jobNodes.get(JobType.GMRCELL.getId()), execNodes);
                        } else {
                            jobNodes.get(JobType.GMR.getId()).add(node);
                            addChildren(node, jobNodes.get(JobType.GMR.getId()), execNodes);
                        }
                    } else {
                        if (LOG.isTraceEnabled())
                            LOG.trace(indent + "Queueing -" + node.toString() + " (code 10)");
                        execNodes.remove(i);
                        finishedNodes.remove(node);
                        queuedNodes.add(node);
                        removeNodesForNextIteration(node, finishedNodes, execNodes, queuedNodes, jobNodes);
                    }
                }
            }
            // next generate MR instructions
            if (!execNodes.isEmpty())
                generateMRJobs(execNodes, inst, writeInst, deleteInst, jobNodes);
            handleSingleOutputJobs(execNodes, jobNodes, finishedNodes);
        }
    }
    // add write and delete inst at the very end.
    // inst.addAll(preWriteDeleteInst);
    inst.addAll(writeInst);
    inst.addAll(deleteInst);
    inst.addAll(endOfBlockInst);
    return inst;
}

Example 100 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class Dag method createIDMapping.

private ArrayList<Lop> createIDMapping(Lop[] nodearray) {
    // Copy sorted nodes into "v" and construct a mapping between Lop IDs and sequence of numbers
    ArrayList<Lop> ret = new ArrayList<>();
    IDMap.clear();
    for (int i = 0; i < nodearray.length; i++) {
        ret.add(nodearray[i]);
        IDMap.put(nodearray[i].getID(), i);
    }
    /*
		 * Compute of All-pair reachability graph (Transitive Closure) of the DAG.
		 * - Perform a depth-first search (DFS) from every node $u$ in the DAG 
		 * - and construct the list of reachable nodes from the node $u$
		 * - store the constructed reachability information in $u$.reachable[] boolean array
		 */
    for (int i = 0; i < nodearray.length; i++) {
        dagDFS(nodearray[i], nodearray[i].create_reachable(nodearray.length));
    }
    // print the nodes in sorted order
    if (LOG.isTraceEnabled()) {
        for (Lop vnode : ret) {
            StringBuilder sb = new StringBuilder();
            sb.append(vnode.getID());
            sb.append("(");
            sb.append(vnode.getLevel());
            sb.append(") ");
            sb.append(vnode.getType());
            sb.append("(");
            for (Lop vin : vnode.getInputs()) {
                sb.append(vin.getID());
                sb.append(",");
            }
            sb.append("), ");
            LOG.trace(sb.toString());
        }
        LOG.trace("topological sort -- done");
    }
    return ret;
}