Examples with Instruction org.apache.sysml.runtime.instructions.Instruction used on opensource projects

Example 41 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project incubator-systemml by apache.

the class ResourceOptimizer method recompileProgramBlock.

private static void recompileProgramBlock(ProgramBlock pb, long cp, long mr) {
    // init compiler memory budget
    InfrastructureAnalyzer.setLocalMaxMemory(cp);
    InfrastructureAnalyzer.setRemoteMaxMemoryMap(mr);
    InfrastructureAnalyzer.setRemoteMaxMemoryReduce(mr);
    // dependent on cp, mr
    OptimizerUtils.resetDefaultSize();
    // recompile instructions (incl predicates)
    if (pb instanceof WhileProgramBlock) {
        WhileProgramBlock wpb = (WhileProgramBlock) pb;
        WhileStatementBlock sb = (WhileStatementBlock) pb.getStatementBlock();
        if (INCLUDE_PREDICATES && sb != null && sb.getPredicateHops() != null) {
            ArrayList<Instruction> inst = Recompiler.recompileHopsDag(sb.getPredicateHops(), new LocalVariableMap(), null, false, false, 0);
            inst = annotateMRJobInstructions(inst, cp, mr);
            wpb.setPredicate(inst);
        }
    } else if (pb instanceof IfProgramBlock) {
        IfProgramBlock ipb = (IfProgramBlock) pb;
        IfStatementBlock sb = (IfStatementBlock) ipb.getStatementBlock();
        if (INCLUDE_PREDICATES && sb != null && sb.getPredicateHops() != null) {
            ArrayList<Instruction> inst = Recompiler.recompileHopsDag(sb.getPredicateHops(), new LocalVariableMap(), null, false, false, 0);
            inst = annotateMRJobInstructions(inst, cp, mr);
            ipb.setPredicate(inst);
        }
    } else if (pb instanceof ForProgramBlock) {
        // incl parfor
        ForProgramBlock fpb = (ForProgramBlock) pb;
        ForStatementBlock sb = (ForStatementBlock) fpb.getStatementBlock();
        if (INCLUDE_PREDICATES && sb != null) {
            if (sb.getFromHops() != null) {
                ArrayList<Instruction> inst = Recompiler.recompileHopsDag(sb.getFromHops(), new LocalVariableMap(), null, false, false, 0);
                inst = annotateMRJobInstructions(inst, cp, mr);
                fpb.setFromInstructions(inst);
            }
            if (sb.getToHops() != null) {
                ArrayList<Instruction> inst = Recompiler.recompileHopsDag(sb.getToHops(), new LocalVariableMap(), null, false, false, 0);
                inst = annotateMRJobInstructions(inst, cp, mr);
                fpb.setToInstructions(inst);
            }
            if (sb.getIncrementHops() != null) {
                ArrayList<Instruction> inst = Recompiler.recompileHopsDag(sb.getIncrementHops(), new LocalVariableMap(), null, false, false, 0);
                inst = annotateMRJobInstructions(inst, cp, mr);
                fpb.setIncrementInstructions(inst);
            }
        }
    } else {
        // last-level program blocks
        StatementBlock sb = pb.getStatementBlock();
        ArrayList<Instruction> inst = Recompiler.recompileHopsDag(sb, sb.getHops(), new LocalVariableMap(), null, false, false, 0);
        inst = annotateMRJobInstructions(inst, cp, mr);
        pb.setInstructions(inst);
    }
    _cntCompilePB++;
}

Example 42 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project systemml by apache.

the class Dag method processConsumers.

private static void processConsumers(Lop node, ArrayList<Instruction> inst, ArrayList<Instruction> deleteInst, Lop locationInfo) {
    // if the count becomes zero, then then variable associated w/ input can be removed
    if (node.removeConsumer() == 0) {
        if (node.getExecLocation() == ExecLocation.Data && ((Data) node).isLiteral()) {
            return;
        }
        String label = node.getOutputParameters().getLabel();
        Instruction currInstr = VariableCPInstruction.prepareRemoveInstruction(label);
        if (locationInfo != null)
            currInstr.setLocation(locationInfo);
        else
            currInstr.setLocation(node);
        inst.add(currInstr);
        excludeRemoveInstruction(label, deleteInst);
    }
}

Example 43 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project systemml by apache.

the class Dag method deleteUpdatedTransientReadVariables.

private static void deleteUpdatedTransientReadVariables(StatementBlock sb, ArrayList<Lop> nodeV, ArrayList<Instruction> inst) {
    if (sb == null)
        return;
    if (LOG.isTraceEnabled())
        LOG.trace("In delete updated variables");
    // CANDIDATE list of variables which could have been updated in this statement block
    HashMap<String, Lop> labelNodeMapping = new HashMap<>();
    // ACTUAL list of variables whose value is updated, AND the old value of the variable
    // is no longer accessible/used.
    HashSet<String> updatedLabels = new HashSet<>();
    HashMap<String, Lop> updatedLabelsLineNum = new HashMap<>();
    // first capture all transient read variables
    for (Lop node : nodeV) {
        if (node.getExecLocation() == ExecLocation.Data && ((Data) node).isTransient() && ((Data) node).getOperationType() == OperationTypes.READ && ((Data) node).getDataType() == DataType.MATRIX) {
            // "node" is considered as updated ONLY IF the old value is not used any more
            // So, make sure that this READ node does not feed into any (transient/persistent) WRITE
            boolean hasWriteParent = false;
            for (Lop p : node.getOutputs()) {
                if (p.getExecLocation() == ExecLocation.Data) {
                    // if the "p" is of type Data, then it has to be a WRITE
                    hasWriteParent = true;
                    break;
                }
            }
            if (!hasWriteParent) {
                // node has no parent of type WRITE, so this is a CANDIDATE variable
                // add it to labelNodeMapping so that it is considered in further processing
                labelNodeMapping.put(node.getOutputParameters().getLabel(), node);
            }
        }
    }
    // capture updated transient write variables
    for (Lop node : nodeV) {
        if (node.getExecLocation() == ExecLocation.Data && ((Data) node).isTransient() && ((Data) node).getOperationType() == OperationTypes.WRITE && ((Data) node).getDataType() == DataType.MATRIX && // check to make sure corresponding (i.e., with the same label/name) transient read is present
        labelNodeMapping.containsKey(node.getOutputParameters().getLabel()) && // check to avoid cases where transient read feeds into a transient write
        !labelNodeMapping.containsValue(node.getInputs().get(0))) {
            updatedLabels.add(node.getOutputParameters().getLabel());
            updatedLabelsLineNum.put(node.getOutputParameters().getLabel(), node);
        }
    }
    // generate RM instructions
    Instruction rm_inst = null;
    for (String label : updatedLabels) {
        rm_inst = VariableCPInstruction.prepareRemoveInstruction(label);
        rm_inst.setLocation(updatedLabelsLineNum.get(label));
        if (LOG.isTraceEnabled())
            LOG.trace(rm_inst.toString());
        inst.add(rm_inst);
    }
}

Example 44 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project systemml by apache.

the class Dag method collapseAssignvarAndRmvarInstructions.

private static ArrayList<Instruction> collapseAssignvarAndRmvarInstructions(ArrayList<Instruction> insts) {
    ArrayList<Instruction> ret = new ArrayList<>();
    Iterator<Instruction> iter = insts.iterator();
    while (iter.hasNext()) {
        Instruction inst = iter.next();
        if (iter.hasNext() && inst instanceof VariableCPInstruction && ((VariableCPInstruction) inst).isAssignVariable()) {
            VariableCPInstruction inst1 = (VariableCPInstruction) inst;
            Instruction inst2 = iter.next();
            if (inst2 instanceof VariableCPInstruction && ((VariableCPInstruction) inst2).isRemoveVariableNoFile() && inst1.getInput1().getName().equals(((VariableCPInstruction) inst2).getInput1().getName())) {
                ret.add(VariableCPInstruction.prepareMoveInstruction(inst1.getInput1().getName(), inst1.getInput2().getName()));
            } else {
                ret.add(inst1);
                ret.add(inst2);
            }
        } else {
            ret.add(inst);
        }
    }
    return ret;
}

Example 45 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project systemml by apache.

the class Dag method generateControlProgramJobs.

/**
 * Method to generate instructions that are executed in Control Program. At
 * this point, this DAG has no dependencies on the MR dag. ie. none of the
 * inputs are outputs of MR jobs
 *
 * @param execNodes list of low-level operators
 * @param inst list of instructions
 * @param writeInst list of write instructions
 * @param deleteInst list of delete instructions
 */
private void generateControlProgramJobs(ArrayList<Lop> execNodes, ArrayList<Instruction> inst, ArrayList<Instruction> writeInst, ArrayList<Instruction> deleteInst) {
    // nodes to be deleted from execnodes
    ArrayList<Lop> markedNodes = new ArrayList<>();
    // variable names to be deleted
    ArrayList<String> var_deletions = new ArrayList<>();
    HashMap<String, Lop> var_deletionsLineNum = new HashMap<>();
    boolean doRmVar = false;
    for (int i = 0; i < execNodes.size(); i++) {
        Lop node = execNodes.get(i);
        doRmVar = false;
        // TODO: statiko -- check if this condition ever evaluated to TRUE
        if (node.getExecLocation() == ExecLocation.Data && ((Data) node).getOperationType() == Data.OperationTypes.READ && ((Data) node).getDataType() == DataType.SCALAR && node.getOutputParameters().getFile_name() == null) {
            markedNodes.add(node);
            continue;
        }
        // output scalar instructions and mark nodes for deletion
        if (node.getExecLocation() == ExecLocation.ControlProgram) {
            if (node.getDataType() == DataType.SCALAR) {
                // Output from lops with SCALAR data type must
                // go into Temporary Variables (Var0, Var1, etc.)
                NodeOutput out = setupNodeOutputs(node, ExecType.CP, false, false);
                // dummy
                inst.addAll(out.getPreInstructions());
                deleteInst.addAll(out.getLastInstructions());
            } else {
                // Output from lops with non-SCALAR data type must
                // go into Temporary Files (temp0, temp1, etc.)
                NodeOutput out = setupNodeOutputs(node, ExecType.CP, false, false);
                inst.addAll(out.getPreInstructions());
                boolean hasTransientWriteParent = false;
                for (Lop parent : node.getOutputs()) {
                    if (parent.getExecLocation() == ExecLocation.Data && ((Data) parent).getOperationType() == Data.OperationTypes.WRITE && ((Data) parent).isTransient()) {
                        hasTransientWriteParent = true;
                        break;
                    }
                }
                if (!hasTransientWriteParent) {
                    deleteInst.addAll(out.getLastInstructions());
                } else {
                    var_deletions.add(node.getOutputParameters().getLabel());
                    var_deletionsLineNum.put(node.getOutputParameters().getLabel(), node);
                }
            }
            String inst_string = "";
            // are handled separately, by simply passing ONLY the output variable to getInstructions()
            if (node.getType() == Lop.Type.ParameterizedBuiltin || node.getType() == Lop.Type.GroupedAgg || node.getType() == Lop.Type.DataGen) {
                inst_string = node.getInstructions(node.getOutputParameters().getLabel());
            } else // separately as well by passing arrays of inputs and outputs
            if (node.getType() == Lop.Type.FunctionCallCP) {
                String[] inputs = new String[node.getInputs().size()];
                String[] outputs = new String[node.getOutputs().size()];
                int count = 0;
                for (Lop in : node.getInputs()) inputs[count++] = in.getOutputParameters().getLabel();
                count = 0;
                for (Lop out : node.getOutputs()) outputs[count++] = out.getOutputParameters().getLabel();
                inst_string = node.getInstructions(inputs, outputs);
            } else if (node.getType() == Lop.Type.Nary) {
                String[] inputs = new String[node.getInputs().size()];
                int count = 0;
                for (Lop in : node.getInputs()) inputs[count++] = in.getOutputParameters().getLabel();
                inst_string = node.getInstructions(inputs, node.getOutputParameters().getLabel());
            } else {
                if (node.getInputs().isEmpty()) {
                    // currently, such a case exists only for Rand lop
                    inst_string = node.getInstructions(node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 1) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 2) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 3 || node.getType() == Type.Ctable) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 4) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getInputs().get(3).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 5) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getInputs().get(3).getOutputParameters().getLabel(), node.getInputs().get(4).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 6) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getInputs().get(3).getOutputParameters().getLabel(), node.getInputs().get(4).getOutputParameters().getLabel(), node.getInputs().get(5).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 7) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getInputs().get(3).getOutputParameters().getLabel(), node.getInputs().get(4).getOutputParameters().getLabel(), node.getInputs().get(5).getOutputParameters().getLabel(), node.getInputs().get(6).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else {
                    String[] inputs = new String[node.getInputs().size()];
                    for (int j = 0; j < node.getInputs().size(); j++) inputs[j] = node.getInputs().get(j).getOutputParameters().getLabel();
                    inst_string = node.getInstructions(inputs, node.getOutputParameters().getLabel());
                }
            }
            try {
                if (LOG.isTraceEnabled())
                    LOG.trace("Generating instruction - " + inst_string);
                Instruction currInstr = InstructionParser.parseSingleInstruction(inst_string);
                if (currInstr == null) {
                    throw new LopsException("Error parsing the instruction:" + inst_string);
                }
                if (node._beginLine != 0)
                    currInstr.setLocation(node);
                else if (!node.getOutputs().isEmpty())
                    currInstr.setLocation(node.getOutputs().get(0));
                else if (!node.getInputs().isEmpty())
                    currInstr.setLocation(node.getInputs().get(0));
                inst.add(currInstr);
            } catch (Exception e) {
                throw new LopsException(node.printErrorLocation() + "Problem generating simple inst - " + inst_string, e);
            }
            markedNodes.add(node);
            doRmVar = true;
        } else if (node.getExecLocation() == ExecLocation.Data) {
            Data dnode = (Data) node;
            Data.OperationTypes op = dnode.getOperationType();
            if (op == Data.OperationTypes.WRITE) {
                NodeOutput out = null;
                if (sendWriteLopToMR(node)) {
                    // In this case, Data WRITE lop goes into MR, and
                    // we don't have to do anything here
                    doRmVar = false;
                } else {
                    out = setupNodeOutputs(node, ExecType.CP, false, false);
                    if (dnode.getDataType() == DataType.SCALAR) {
                        // processing is same for both transient and persistent scalar writes
                        writeInst.addAll(out.getLastInstructions());
                        doRmVar = false;
                    } else {
                        // setupNodeOutputs() handles both transient and persistent matrix writes
                        if (dnode.isTransient()) {
                            deleteInst.addAll(out.getLastInstructions());
                            doRmVar = false;
                        } else {
                            // In case of persistent write lop, write instruction will be generated
                            // and that instruction must be added to <code>inst</code> so that it gets
                            // executed immediately. If it is added to <code>deleteInst</code> then it
                            // gets executed at the end of program block's execution
                            inst.addAll(out.getLastInstructions());
                            doRmVar = true;
                        }
                    }
                    markedNodes.add(node);
                }
            } else {
                // generate a temp label to hold the value that is read from HDFS
                if (node.getDataType() == DataType.SCALAR) {
                    node.getOutputParameters().setLabel(Lop.SCALAR_VAR_NAME_PREFIX + var_index.getNextID());
                    String io_inst = node.getInstructions(node.getOutputParameters().getLabel(), node.getOutputParameters().getFile_name());
                    CPInstruction currInstr = CPInstructionParser.parseSingleInstruction(io_inst);
                    currInstr.setLocation(node);
                    inst.add(currInstr);
                    Instruction tempInstr = VariableCPInstruction.prepareRemoveInstruction(node.getOutputParameters().getLabel());
                    tempInstr.setLocation(node);
                    deleteInst.add(tempInstr);
                } else {
                    throw new LopsException("Matrix READs are not handled in CP yet!");
                }
                markedNodes.add(node);
                doRmVar = true;
            }
        }
        // see if rmvar instructions can be generated for node's inputs
        if (doRmVar)
            processConsumersForInputs(node, inst, deleteInst);
        doRmVar = false;
    }
    for (String var : var_deletions) {
        Instruction rmInst = VariableCPInstruction.prepareRemoveInstruction(var);
        if (LOG.isTraceEnabled())
            LOG.trace("  Adding var_deletions: " + rmInst.toString());
        rmInst.setLocation(var_deletionsLineNum.get(var));
        deleteInst.add(rmInst);
    }
    // delete all marked nodes
    for (Lop node : markedNodes) {
        execNodes.remove(node);
    }
}