Examples with Instruction org.apache.sysml.runtime.instructions.Instruction used on opensource projects

Example 31 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project incubator-systemml by apache.

the class Recompiler method recompileForPredicates.

private static void recompileForPredicates(ForProgramBlock fpb, ForStatementBlock fsb, LocalVariableMap vars, RecompileStatus status, long tid, ResetType resetRecompile) {
    if (fsb != null) {
        Hop fromHops = fsb.getFromHops();
        Hop toHops = fsb.getToHops();
        Hop incrHops = fsb.getIncrementHops();
        // handle recompilation flags
        if (ParForProgramBlock.RESET_RECOMPILATION_FLAGs && resetRecompile.isReset()) {
            if (fromHops != null) {
                ArrayList<Instruction> tmp = recompileHopsDag(fromHops, vars, status, true, false, tid);
                fpb.setFromInstructions(tmp);
                Hop.resetRecompilationFlag(fromHops, ExecType.CP, resetRecompile);
            }
            if (toHops != null) {
                ArrayList<Instruction> tmp = recompileHopsDag(toHops, vars, status, true, false, tid);
                fpb.setToInstructions(tmp);
                Hop.resetRecompilationFlag(toHops, ExecType.CP, resetRecompile);
            }
            if (incrHops != null) {
                ArrayList<Instruction> tmp = recompileHopsDag(incrHops, vars, status, true, false, tid);
                fpb.setIncrementInstructions(tmp);
                Hop.resetRecompilationFlag(incrHops, ExecType.CP, resetRecompile);
            }
            fsb.updatePredicateRecompilationFlags();
        } else // no reset of recompilation flags
        {
            if (fromHops != null) {
                ArrayList<Instruction> tmp = recompileHopsDag(fromHops, vars, status, true, false, tid);
                fpb.setFromInstructions(tmp);
            }
            if (toHops != null) {
                ArrayList<Instruction> tmp = recompileHopsDag(toHops, vars, status, true, false, tid);
                fpb.setToInstructions(tmp);
            }
            if (incrHops != null) {
                ArrayList<Instruction> tmp = recompileHopsDag(incrHops, vars, status, true, false, tid);
                fpb.setIncrementInstructions(tmp);
            }
        }
    }
}

Example 32 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project incubator-systemml by apache.

the class Recompiler method recompileWhilePredicate.

private static void recompileWhilePredicate(WhileProgramBlock wpb, WhileStatementBlock wsb, LocalVariableMap vars, RecompileStatus status, long tid, ResetType resetRecompile) {
    if (wsb == null)
        return;
    Hop hops = wsb.getPredicateHops();
    if (hops != null) {
        ArrayList<Instruction> tmp = recompileHopsDag(hops, vars, status, true, false, tid);
        wpb.setPredicate(tmp);
        if (ParForProgramBlock.RESET_RECOMPILATION_FLAGs && resetRecompile.isReset()) {
            Hop.resetRecompilationFlag(hops, ExecType.CP, resetRecompile);
            wsb.updatePredicateRecompilationFlag();
        }
    }
}

Example 33 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project incubator-systemml by apache.

the class Dag method generateControlProgramJobs.

/**
 * Method to generate instructions that are executed in Control Program. At
 * this point, this DAG has no dependencies on the MR dag. ie. none of the
 * inputs are outputs of MR jobs
 *
 * @param execNodes list of low-level operators
 * @param inst list of instructions
 * @param writeInst list of write instructions
 * @param deleteInst list of delete instructions
 */
private void generateControlProgramJobs(ArrayList<Lop> execNodes, ArrayList<Instruction> inst, ArrayList<Instruction> writeInst, ArrayList<Instruction> deleteInst) {
    // nodes to be deleted from execnodes
    ArrayList<Lop> markedNodes = new ArrayList<>();
    // variable names to be deleted
    ArrayList<String> var_deletions = new ArrayList<>();
    HashMap<String, Lop> var_deletionsLineNum = new HashMap<>();
    boolean doRmVar = false;
    for (int i = 0; i < execNodes.size(); i++) {
        Lop node = execNodes.get(i);
        doRmVar = false;
        // TODO: statiko -- check if this condition ever evaluated to TRUE
        if (node.getExecLocation() == ExecLocation.Data && ((Data) node).getOperationType() == Data.OperationTypes.READ && ((Data) node).getDataType() == DataType.SCALAR && node.getOutputParameters().getFile_name() == null) {
            markedNodes.add(node);
            continue;
        }
        // output scalar instructions and mark nodes for deletion
        if (node.getExecLocation() == ExecLocation.ControlProgram) {
            if (node.getDataType() == DataType.SCALAR) {
                // Output from lops with SCALAR data type must
                // go into Temporary Variables (Var0, Var1, etc.)
                NodeOutput out = setupNodeOutputs(node, ExecType.CP, false, false);
                // dummy
                inst.addAll(out.getPreInstructions());
                deleteInst.addAll(out.getLastInstructions());
            } else {
                // Output from lops with non-SCALAR data type must
                // go into Temporary Files (temp0, temp1, etc.)
                NodeOutput out = setupNodeOutputs(node, ExecType.CP, false, false);
                inst.addAll(out.getPreInstructions());
                boolean hasTransientWriteParent = false;
                for (Lop parent : node.getOutputs()) {
                    if (parent.getExecLocation() == ExecLocation.Data && ((Data) parent).getOperationType() == Data.OperationTypes.WRITE && ((Data) parent).isTransient()) {
                        hasTransientWriteParent = true;
                        break;
                    }
                }
                if (!hasTransientWriteParent) {
                    deleteInst.addAll(out.getLastInstructions());
                } else {
                    var_deletions.add(node.getOutputParameters().getLabel());
                    var_deletionsLineNum.put(node.getOutputParameters().getLabel(), node);
                }
            }
            String inst_string = "";
            // are handled separately, by simply passing ONLY the output variable to getInstructions()
            if (node.getType() == Lop.Type.ParameterizedBuiltin || node.getType() == Lop.Type.GroupedAgg || node.getType() == Lop.Type.DataGen) {
                inst_string = node.getInstructions(node.getOutputParameters().getLabel());
            } else // separately as well by passing arrays of inputs and outputs
            if (node.getType() == Lop.Type.FunctionCallCP) {
                String[] inputs = new String[node.getInputs().size()];
                String[] outputs = new String[node.getOutputs().size()];
                int count = 0;
                for (Lop in : node.getInputs()) inputs[count++] = in.getOutputParameters().getLabel();
                count = 0;
                for (Lop out : node.getOutputs()) outputs[count++] = out.getOutputParameters().getLabel();
                inst_string = node.getInstructions(inputs, outputs);
            } else if (node.getType() == Lop.Type.Nary) {
                String[] inputs = new String[node.getInputs().size()];
                int count = 0;
                for (Lop in : node.getInputs()) inputs[count++] = in.getOutputParameters().getLabel();
                inst_string = node.getInstructions(inputs, node.getOutputParameters().getLabel());
            } else {
                if (node.getInputs().isEmpty()) {
                    // currently, such a case exists only for Rand lop
                    inst_string = node.getInstructions(node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 1) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 2) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 3 || node.getType() == Type.Ctable) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 4) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getInputs().get(3).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 5) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getInputs().get(3).getOutputParameters().getLabel(), node.getInputs().get(4).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 6) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getInputs().get(3).getOutputParameters().getLabel(), node.getInputs().get(4).getOutputParameters().getLabel(), node.getInputs().get(5).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 7) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getInputs().get(3).getOutputParameters().getLabel(), node.getInputs().get(4).getOutputParameters().getLabel(), node.getInputs().get(5).getOutputParameters().getLabel(), node.getInputs().get(6).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else {
                    String[] inputs = new String[node.getInputs().size()];
                    for (int j = 0; j < node.getInputs().size(); j++) inputs[j] = node.getInputs().get(j).getOutputParameters().getLabel();
                    inst_string = node.getInstructions(inputs, node.getOutputParameters().getLabel());
                }
            }
            try {
                if (LOG.isTraceEnabled())
                    LOG.trace("Generating instruction - " + inst_string);
                Instruction currInstr = InstructionParser.parseSingleInstruction(inst_string);
                if (currInstr == null) {
                    throw new LopsException("Error parsing the instruction:" + inst_string);
                }
                if (node._beginLine != 0)
                    currInstr.setLocation(node);
                else if (!node.getOutputs().isEmpty())
                    currInstr.setLocation(node.getOutputs().get(0));
                else if (!node.getInputs().isEmpty())
                    currInstr.setLocation(node.getInputs().get(0));
                inst.add(currInstr);
            } catch (Exception e) {
                throw new LopsException(node.printErrorLocation() + "Problem generating simple inst - " + inst_string, e);
            }
            markedNodes.add(node);
            doRmVar = true;
        } else if (node.getExecLocation() == ExecLocation.Data) {
            Data dnode = (Data) node;
            Data.OperationTypes op = dnode.getOperationType();
            if (op == Data.OperationTypes.WRITE) {
                NodeOutput out = null;
                if (sendWriteLopToMR(node)) {
                    // In this case, Data WRITE lop goes into MR, and
                    // we don't have to do anything here
                    doRmVar = false;
                } else {
                    out = setupNodeOutputs(node, ExecType.CP, false, false);
                    if (dnode.getDataType() == DataType.SCALAR) {
                        // processing is same for both transient and persistent scalar writes
                        writeInst.addAll(out.getLastInstructions());
                        doRmVar = false;
                    } else {
                        // setupNodeOutputs() handles both transient and persistent matrix writes
                        if (dnode.isTransient()) {
                            deleteInst.addAll(out.getLastInstructions());
                            doRmVar = false;
                        } else {
                            // In case of persistent write lop, write instruction will be generated
                            // and that instruction must be added to <code>inst</code> so that it gets
                            // executed immediately. If it is added to <code>deleteInst</code> then it
                            // gets executed at the end of program block's execution
                            inst.addAll(out.getLastInstructions());
                            doRmVar = true;
                        }
                    }
                    markedNodes.add(node);
                }
            } else {
                // generate a temp label to hold the value that is read from HDFS
                if (node.getDataType() == DataType.SCALAR) {
                    node.getOutputParameters().setLabel(Lop.SCALAR_VAR_NAME_PREFIX + var_index.getNextID());
                    String io_inst = node.getInstructions(node.getOutputParameters().getLabel(), node.getOutputParameters().getFile_name());
                    CPInstruction currInstr = CPInstructionParser.parseSingleInstruction(io_inst);
                    currInstr.setLocation(node);
                    inst.add(currInstr);
                    Instruction tempInstr = VariableCPInstruction.prepareRemoveInstruction(node.getOutputParameters().getLabel());
                    tempInstr.setLocation(node);
                    deleteInst.add(tempInstr);
                } else {
                    throw new LopsException("Matrix READs are not handled in CP yet!");
                }
                markedNodes.add(node);
                doRmVar = true;
            }
        }
        // see if rmvar instructions can be generated for node's inputs
        if (doRmVar)
            processConsumersForInputs(node, inst, deleteInst);
        doRmVar = false;
    }
    for (String var : var_deletions) {
        Instruction rmInst = VariableCPInstruction.prepareRemoveInstruction(var);
        if (LOG.isTraceEnabled())
            LOG.trace("  Adding var_deletions: " + rmInst.toString());
        rmInst.setLocation(var_deletionsLineNum.get(var));
        deleteInst.add(rmInst);
    }
    // delete all marked nodes
    for (Lop node : markedNodes) {
        execNodes.remove(node);
    }
}

Example 34 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project incubator-systemml by apache.

the class Dag method generateRemoveInstructions.

private static void generateRemoveInstructions(StatementBlock sb, ArrayList<Instruction> deleteInst) {
    if (sb == null)
        return;
    if (LOG.isTraceEnabled())
        LOG.trace("In generateRemoveInstructions()");
    Instruction inst = null;
    // (currently required for specific cases of external functions)
    for (String varName : sb.liveIn().getVariableNames()) {
        if (!sb.liveOut().containsVariable(varName)) {
            inst = VariableCPInstruction.prepareRemoveInstruction(varName);
            inst.setLocation(sb.getFilename(), sb.getEndLine(), sb.getEndLine(), -1, -1);
            deleteInst.add(inst);
            if (LOG.isTraceEnabled())
                LOG.trace("  Adding " + inst.toString());
        }
    }
}

Example 35 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project incubator-systemml by apache.

the class Dag method collapseAssignvarAndRmvarInstructions.

private static ArrayList<Instruction> collapseAssignvarAndRmvarInstructions(ArrayList<Instruction> insts) {
    ArrayList<Instruction> ret = new ArrayList<>();
    Iterator<Instruction> iter = insts.iterator();
    while (iter.hasNext()) {
        Instruction inst = iter.next();
        if (iter.hasNext() && inst instanceof VariableCPInstruction && ((VariableCPInstruction) inst).isAssignVariable()) {
            VariableCPInstruction inst1 = (VariableCPInstruction) inst;
            Instruction inst2 = iter.next();
            if (inst2 instanceof VariableCPInstruction && ((VariableCPInstruction) inst2).isRemoveVariableNoFile() && inst1.getInput1().getName().equals(((VariableCPInstruction) inst2).getInput1().getName())) {
                ret.add(VariableCPInstruction.prepareMoveInstruction(inst1.getInput1().getName(), inst1.getInput2().getName()));
            } else {
                ret.add(inst1);
                ret.add(inst2);
            }
        } else {
            ret.add(inst);
        }
    }
    return ret;
}