Examples with Lop org.apache.sysml.lops.Lop used on opensource projects

Example 46 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class DMLTranslator method printLops.

public void printLops(StatementBlock current) {
    if (LOG.isDebugEnabled()) {
        ArrayList<Lop> lopsDAG = current.getLops();
        LOG.debug("\n********************** LOPS DAG FOR BLOCK *******************");
        if (current instanceof FunctionStatementBlock) {
            if (current.getNumStatements() > 1)
                LOG.debug("Function statement block has more than 1 stmt");
            FunctionStatement fstmt = (FunctionStatement) current.getStatement(0);
            for (StatementBlock child : fstmt.getBody()) {
                printLops(child);
            }
        }
        if (current instanceof WhileStatementBlock) {
            // print predicate lops
            WhileStatementBlock wstb = (WhileStatementBlock) current;
            Hop predicateHops = ((WhileStatementBlock) current).getPredicateHops();
            LOG.debug("\n********************** PREDICATE LOPS *******************");
            Lop predicateLops = predicateHops.getLops();
            if (predicateLops == null)
                predicateLops = predicateHops.constructLops();
            predicateLops.printMe();
            if (wstb.getNumStatements() > 1) {
                LOG.error(wstb.printBlockErrorLocation() + "WhileStatementBlock has more than 1 statement");
                throw new HopsException(wstb.printBlockErrorLocation() + "WhileStatementBlock has more than 1 statement");
            }
            WhileStatement ws = (WhileStatement) wstb.getStatement(0);
            for (StatementBlock sb : ws.getBody()) {
                printLops(sb);
            }
        }
        if (current instanceof IfStatementBlock) {
            // print predicate lops
            IfStatementBlock istb = (IfStatementBlock) current;
            Hop predicateHops = ((IfStatementBlock) current).getPredicateHops();
            LOG.debug("\n********************** PREDICATE LOPS *******************");
            Lop predicateLops = predicateHops.getLops();
            if (predicateLops == null)
                predicateLops = predicateHops.constructLops();
            predicateLops.printMe();
            if (istb.getNumStatements() > 1) {
                LOG.error(istb.printBlockErrorLocation() + "IfStatmentBlock has more than 1 statement");
                throw new HopsException(istb.printBlockErrorLocation() + "IfStatmentBlock has more than 1 statement");
            }
            IfStatement is = (IfStatement) istb.getStatement(0);
            LOG.debug("\n**** LOPS DAG FOR IF BODY ****");
            for (StatementBlock sb : is.getIfBody()) {
                printLops(sb);
            }
            if (!is.getElseBody().isEmpty()) {
                LOG.debug("\n**** LOPS DAG FOR IF BODY ****");
                for (StatementBlock sb : is.getElseBody()) {
                    printLops(sb);
                }
            }
        }
        if (current instanceof ForStatementBlock) {
            // print predicate lops
            ForStatementBlock fsb = (ForStatementBlock) current;
            LOG.debug("\n********************** PREDICATE LOPS *******************");
            if (fsb.getFromHops() != null) {
                LOG.debug("FROM:");
                Lop llops = fsb.getFromLops();
                if (llops == null)
                    llops = fsb.getFromHops().constructLops();
                llops.printMe();
            }
            if (fsb.getToHops() != null) {
                LOG.debug("TO:");
                Lop llops = fsb.getToLops();
                if (llops == null)
                    llops = fsb.getToHops().constructLops();
                llops.printMe();
            }
            if (fsb.getIncrementHops() != null) {
                LOG.debug("INCREMENT:");
                Lop llops = fsb.getIncrementLops();
                if (llops == null)
                    llops = fsb.getIncrementHops().constructLops();
                llops.printMe();
            }
            if (fsb.getNumStatements() > 1) {
                LOG.error(fsb.printBlockErrorLocation() + "ForStatementBlock has more than 1 statement");
                throw new HopsException(fsb.printBlockErrorLocation() + "ForStatementBlock has more than 1 statement");
            }
            ForStatement ws = (ForStatement) fsb.getStatement(0);
            for (StatementBlock sb : ws.getBody()) {
                printLops(sb);
            }
        }
        if (lopsDAG != null && !lopsDAG.isEmpty()) {
            Iterator<Lop> iter = lopsDAG.iterator();
            while (iter.hasNext()) {
                LOG.debug("\n********************** OUTPUT LOPS *******************");
                iter.next().printMe();
            }
        }
    }
}

Example 47 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class Dag method checkDataGenAsChildNode.

private boolean checkDataGenAsChildNode(Lop node, ArrayList<Lop> nodesVec) {
    if (nodesVec.isEmpty())
        return true;
    int index = IDMap.get(node.getID());
    boolean onlyDatagen = true;
    for (Lop n : nodesVec) {
        if (n.definesMRJob() && n.get_reachable()[index] && JobType.findJobTypeFromLop(n) != JobType.DATAGEN)
            onlyDatagen = false;
    }
    // return true also when there is no lop in "nodesVec" that defines a MR job.
    return onlyDatagen;
}

Example 48 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class Dag method getRecordReaderInstructions.

/**
 * Method to get record reader instructions for a MR job.
 *
 * @param node low-level operator
 * @param execNodes list of exec nodes
 * @param inputStrings list of input strings
 * @param recordReaderInstructions list of record reader instructions
 * @param nodeIndexMapping node index mapping
 * @param start_index start index
 * @param inputLabels list of input labels
 * @param inputLops list of input lops
 * @param MRJobLineNumbers MR job line numbers
 * @return -1 if problem
 */
private static int getRecordReaderInstructions(Lop node, ArrayList<Lop> execNodes, ArrayList<String> inputStrings, ArrayList<String> recordReaderInstructions, HashMap<Lop, Integer> nodeIndexMapping, int[] start_index, ArrayList<String> inputLabels, ArrayList<Lop> inputLops, ArrayList<Integer> MRJobLineNumbers) {
    // if input source, return index
    if (nodeIndexMapping.containsKey(node))
        return nodeIndexMapping.get(node);
    // not input source and not in exec nodes, then return.
    if (!execNodes.contains(node))
        return -1;
    ArrayList<Integer> inputIndices = new ArrayList<>();
    int max_input_index = -1;
    // get mapper instructions
    for (int i = 0; i < node.getInputs().size(); i++) {
        // recurse
        Lop childNode = node.getInputs().get(i);
        int ret_val = getRecordReaderInstructions(childNode, execNodes, inputStrings, recordReaderInstructions, nodeIndexMapping, start_index, inputLabels, inputLops, MRJobLineNumbers);
        inputIndices.add(ret_val);
        if (ret_val > max_input_index) {
            max_input_index = ret_val;
        // child_for_max_input_index = childNode;
        }
    }
    // instructions
    if ((node.getExecLocation() == ExecLocation.RecordReader)) {
        int output_index = max_input_index;
        // cannot reuse index if this is true
        // need to add better indexing schemes
        output_index = start_index[0];
        start_index[0]++;
        nodeIndexMapping.put(node, output_index);
        // only Ranagepick lop can contribute to labels
        if (node.getType() == Type.PickValues) {
            PickByCount pbc = (PickByCount) node;
            if (pbc.getOperationType() == PickByCount.OperationTypes.RANGEPICK) {
                // always the second input is a scalar
                int scalarIndex = 1;
                // if data lop not a literal -- add label
                if (node.getInputs().get(scalarIndex).getExecLocation() == ExecLocation.Data && !((Data) (node.getInputs().get(scalarIndex))).isLiteral()) {
                    inputLabels.add(node.getInputs().get(scalarIndex).getOutputParameters().getLabel());
                    inputLops.add(node.getInputs().get(scalarIndex));
                }
                // if not data lop, then this is an intermediate variable.
                if (node.getInputs().get(scalarIndex).getExecLocation() != ExecLocation.Data) {
                    inputLabels.add(node.getInputs().get(scalarIndex).getOutputParameters().getLabel());
                    inputLops.add(node.getInputs().get(scalarIndex));
                }
            }
        }
        // get recordreader instruction.
        if (node.getInputs().size() == 2) {
            recordReaderInstructions.add(node.getInstructions(inputIndices.get(0), inputIndices.get(1), output_index));
            if (DMLScript.ENABLE_DEBUG_MODE) {
                MRJobLineNumbers.add(node._beginLine);
            }
        } else
            throw new LopsException("Unexpected number of inputs while generating a RecordReader Instruction");
        return output_index;
    }
    return -1;
}

Example 49 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class Dag method generateInstructionsForInputVariables.

/**
 * Method to generate createvar instructions, which creates a new entry
 * in the symbol table. One instruction is generated for every LOP that is
 * 1) type Data and
 * 2) persistent and
 * 3) matrix and
 * 4) read
 *
 * Transient reads needn't be considered here since the previous program
 * block would already create appropriate entries in the symbol table.
 *
 * @param nodes_v list of nodes
 * @param inst list of instructions
 */
private static void generateInstructionsForInputVariables(ArrayList<Lop> nodes_v, ArrayList<Instruction> inst) {
    for (Lop n : nodes_v) {
        if (n.getExecLocation() == ExecLocation.Data && !((Data) n).isTransient() && ((Data) n).getOperationType() == OperationTypes.READ && (n.getDataType() == DataType.MATRIX || n.getDataType() == DataType.FRAME)) {
            if (!((Data) n).isLiteral()) {
                try {
                    String inst_string = n.getInstructions();
                    CPInstruction currInstr = CPInstructionParser.parseSingleInstruction(inst_string);
                    currInstr.setLocation(n);
                    inst.add(currInstr);
                } catch (DMLRuntimeException e) {
                    throw new LopsException(n.printErrorLocation() + "error generating instructions from input variables in Dag -- \n", e);
                }
            }
        }
    }
}

Example 50 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class Dag method deleteUpdatedTransientReadVariables.

private static void deleteUpdatedTransientReadVariables(StatementBlock sb, ArrayList<Lop> nodeV, ArrayList<Instruction> inst) {
    if (sb == null)
        return;
    if (LOG.isTraceEnabled())
        LOG.trace("In delete updated variables");
    // CANDIDATE list of variables which could have been updated in this statement block
    HashMap<String, Lop> labelNodeMapping = new HashMap<>();
    // ACTUAL list of variables whose value is updated, AND the old value of the variable
    // is no longer accessible/used.
    HashSet<String> updatedLabels = new HashSet<>();
    HashMap<String, Lop> updatedLabelsLineNum = new HashMap<>();
    // first capture all transient read variables
    for (Lop node : nodeV) {
        if (node.getExecLocation() == ExecLocation.Data && ((Data) node).isTransient() && ((Data) node).getOperationType() == OperationTypes.READ && ((Data) node).getDataType() == DataType.MATRIX) {
            // "node" is considered as updated ONLY IF the old value is not used any more
            // So, make sure that this READ node does not feed into any (transient/persistent) WRITE
            boolean hasWriteParent = false;
            for (Lop p : node.getOutputs()) {
                if (p.getExecLocation() == ExecLocation.Data) {
                    // if the "p" is of type Data, then it has to be a WRITE
                    hasWriteParent = true;
                    break;
                }
            }
            if (!hasWriteParent) {
                // node has no parent of type WRITE, so this is a CANDIDATE variable
                // add it to labelNodeMapping so that it is considered in further processing
                labelNodeMapping.put(node.getOutputParameters().getLabel(), node);
            }
        }
    }
    // capture updated transient write variables
    for (Lop node : nodeV) {
        if (node.getExecLocation() == ExecLocation.Data && ((Data) node).isTransient() && ((Data) node).getOperationType() == OperationTypes.WRITE && ((Data) node).getDataType() == DataType.MATRIX && // check to make sure corresponding (i.e., with the same label/name) transient read is present
        labelNodeMapping.containsKey(node.getOutputParameters().getLabel()) && // check to avoid cases where transient read feeds into a transient write
        !labelNodeMapping.containsValue(node.getInputs().get(0))) {
            updatedLabels.add(node.getOutputParameters().getLabel());
            updatedLabelsLineNum.put(node.getOutputParameters().getLabel(), node);
        }
    }
    // generate RM instructions
    Instruction rm_inst = null;
    for (String label : updatedLabels) {
        rm_inst = VariableCPInstruction.prepareRemoveInstruction(label);
        rm_inst.setLocation(updatedLabelsLineNum.get(label));
        if (LOG.isTraceEnabled())
            LOG.trace(rm_inst.toString());
        inst.add(rm_inst);
    }
}