Examples with CPInstruction org.apache.sysml.runtime.instructions.cp.CPInstruction used on opensource projects

Example 6 with CPInstruction

use of org.apache.sysml.runtime.instructions.cp.CPInstruction in project incubator-systemml by apache.

the class CostEstimatorStaticRuntime method getCPInstTimeEstimate.

@Override
@SuppressWarnings("unused")
protected double getCPInstTimeEstimate(Instruction inst, VarStats[] vs, String[] args) {
    CPInstruction cpinst = (CPInstruction) inst;
    // load time into mem
    double ltime = 0;
    if (!vs[0]._inmem) {
        ltime += getHDFSReadTime(vs[0]._rlen, vs[0]._clen, vs[0].getSparsity());
        // eviction costs
        if (CacheableData.CACHING_WRITE_CACHE_ON_READ && LazyWriteBuffer.getWriteBufferLimit() < MatrixBlock.estimateSizeOnDisk(vs[0]._rlen, vs[0]._clen, (long) ((vs[0]._nnz < 0) ? vs[0]._rlen * vs[0]._clen : vs[0]._nnz))) {
            ltime += Math.abs(getFSWriteTime(vs[0]._rlen, vs[0]._clen, vs[0].getSparsity()));
        }
        vs[0]._inmem = true;
    }
    if (!vs[1]._inmem) {
        ltime += getHDFSReadTime(vs[1]._rlen, vs[1]._clen, vs[1].getSparsity());
        // eviction costs
        if (CacheableData.CACHING_WRITE_CACHE_ON_READ && LazyWriteBuffer.getWriteBufferLimit() < MatrixBlock.estimateSizeOnDisk(vs[1]._rlen, vs[1]._clen, (long) ((vs[1]._nnz < 0) ? vs[1]._rlen * vs[1]._clen : vs[1]._nnz))) {
            ltime += Math.abs(getFSWriteTime(vs[1]._rlen, vs[1]._clen, vs[1].getSparsity()));
        }
        vs[1]._inmem = true;
    }
    if (LOG.isDebugEnabled() && ltime != 0) {
        LOG.debug("Cost[" + cpinst.getOpcode() + " - read] = " + ltime);
    }
    // exec time CP instruction
    String opcode = (cpinst instanceof FunctionCallCPInstruction) ? InstructionUtils.getOpCode(cpinst.toString()) : cpinst.getOpcode();
    double etime = getInstTimeEstimate(opcode, vs, args, ExecType.CP);
    // write time caching
    double wtime = 0;
    // double wtime = getFSWriteTime( vs[2]._rlen, vs[2]._clen, (vs[2]._nnz<0)? 1.0:(double)vs[2]._nnz/vs[2]._rlen/vs[2]._clen );
    if (inst instanceof VariableCPInstruction && ((VariableCPInstruction) inst).getOpcode().equals("write"))
        wtime += getHDFSWriteTime(vs[2]._rlen, vs[2]._clen, vs[2].getSparsity(), ((VariableCPInstruction) inst).getInput3().getName());
    if (LOG.isDebugEnabled() && wtime != 0) {
        LOG.debug("Cost[" + cpinst.getOpcode() + " - write] = " + wtime);
    }
    // total costs
    double costs = ltime + etime + wtime;
    return costs;
}

Example 7 with CPInstruction

use of org.apache.sysml.runtime.instructions.cp.CPInstruction in project incubator-systemml by apache.

the class Dag method generateControlProgramJobs.

/**
 * Method to generate instructions that are executed in Control Program. At
 * this point, this DAG has no dependencies on the MR dag. ie. none of the
 * inputs are outputs of MR jobs
 *
 * @param execNodes list of low-level operators
 * @param inst list of instructions
 * @param writeInst list of write instructions
 * @param deleteInst list of delete instructions
 */
private void generateControlProgramJobs(ArrayList<Lop> execNodes, ArrayList<Instruction> inst, ArrayList<Instruction> writeInst, ArrayList<Instruction> deleteInst) {
    // nodes to be deleted from execnodes
    ArrayList<Lop> markedNodes = new ArrayList<>();
    // variable names to be deleted
    ArrayList<String> var_deletions = new ArrayList<>();
    HashMap<String, Lop> var_deletionsLineNum = new HashMap<>();
    boolean doRmVar = false;
    for (int i = 0; i < execNodes.size(); i++) {
        Lop node = execNodes.get(i);
        doRmVar = false;
        // TODO: statiko -- check if this condition ever evaluated to TRUE
        if (node.getExecLocation() == ExecLocation.Data && ((Data) node).getOperationType() == Data.OperationTypes.READ && ((Data) node).getDataType() == DataType.SCALAR && node.getOutputParameters().getFile_name() == null) {
            markedNodes.add(node);
            continue;
        }
        // output scalar instructions and mark nodes for deletion
        if (node.getExecLocation() == ExecLocation.ControlProgram) {
            if (node.getDataType() == DataType.SCALAR) {
                // Output from lops with SCALAR data type must
                // go into Temporary Variables (Var0, Var1, etc.)
                NodeOutput out = setupNodeOutputs(node, ExecType.CP, false, false);
                // dummy
                inst.addAll(out.getPreInstructions());
                deleteInst.addAll(out.getLastInstructions());
            } else {
                // Output from lops with non-SCALAR data type must
                // go into Temporary Files (temp0, temp1, etc.)
                NodeOutput out = setupNodeOutputs(node, ExecType.CP, false, false);
                inst.addAll(out.getPreInstructions());
                boolean hasTransientWriteParent = false;
                for (Lop parent : node.getOutputs()) {
                    if (parent.getExecLocation() == ExecLocation.Data && ((Data) parent).getOperationType() == Data.OperationTypes.WRITE && ((Data) parent).isTransient()) {
                        hasTransientWriteParent = true;
                        break;
                    }
                }
                if (!hasTransientWriteParent) {
                    deleteInst.addAll(out.getLastInstructions());
                } else {
                    var_deletions.add(node.getOutputParameters().getLabel());
                    var_deletionsLineNum.put(node.getOutputParameters().getLabel(), node);
                }
            }
            String inst_string = "";
            // are handled separately, by simply passing ONLY the output variable to getInstructions()
            if (node.getType() == Lop.Type.ParameterizedBuiltin || node.getType() == Lop.Type.GroupedAgg || node.getType() == Lop.Type.DataGen) {
                inst_string = node.getInstructions(node.getOutputParameters().getLabel());
            } else // separately as well by passing arrays of inputs and outputs
            if (node.getType() == Lop.Type.FunctionCallCP) {
                String[] inputs = new String[node.getInputs().size()];
                String[] outputs = new String[node.getOutputs().size()];
                int count = 0;
                for (Lop in : node.getInputs()) inputs[count++] = in.getOutputParameters().getLabel();
                count = 0;
                for (Lop out : node.getOutputs()) outputs[count++] = out.getOutputParameters().getLabel();
                inst_string = node.getInstructions(inputs, outputs);
            } else if (node.getType() == Lop.Type.Nary) {
                String[] inputs = new String[node.getInputs().size()];
                int count = 0;
                for (Lop in : node.getInputs()) inputs[count++] = in.getOutputParameters().getLabel();
                inst_string = node.getInstructions(inputs, node.getOutputParameters().getLabel());
            } else {
                if (node.getInputs().isEmpty()) {
                    // currently, such a case exists only for Rand lop
                    inst_string = node.getInstructions(node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 1) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 2) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 3 || node.getType() == Type.Ctable) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 4) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getInputs().get(3).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 5) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getInputs().get(3).getOutputParameters().getLabel(), node.getInputs().get(4).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 6) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getInputs().get(3).getOutputParameters().getLabel(), node.getInputs().get(4).getOutputParameters().getLabel(), node.getInputs().get(5).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 7) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getInputs().get(3).getOutputParameters().getLabel(), node.getInputs().get(4).getOutputParameters().getLabel(), node.getInputs().get(5).getOutputParameters().getLabel(), node.getInputs().get(6).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else {
                    String[] inputs = new String[node.getInputs().size()];
                    for (int j = 0; j < node.getInputs().size(); j++) inputs[j] = node.getInputs().get(j).getOutputParameters().getLabel();
                    inst_string = node.getInstructions(inputs, node.getOutputParameters().getLabel());
                }
            }
            try {
                if (LOG.isTraceEnabled())
                    LOG.trace("Generating instruction - " + inst_string);
                Instruction currInstr = InstructionParser.parseSingleInstruction(inst_string);
                if (currInstr == null) {
                    throw new LopsException("Error parsing the instruction:" + inst_string);
                }
                if (node._beginLine != 0)
                    currInstr.setLocation(node);
                else if (!node.getOutputs().isEmpty())
                    currInstr.setLocation(node.getOutputs().get(0));
                else if (!node.getInputs().isEmpty())
                    currInstr.setLocation(node.getInputs().get(0));
                inst.add(currInstr);
            } catch (Exception e) {
                throw new LopsException(node.printErrorLocation() + "Problem generating simple inst - " + inst_string, e);
            }
            markedNodes.add(node);
            doRmVar = true;
        } else if (node.getExecLocation() == ExecLocation.Data) {
            Data dnode = (Data) node;
            Data.OperationTypes op = dnode.getOperationType();
            if (op == Data.OperationTypes.WRITE) {
                NodeOutput out = null;
                if (sendWriteLopToMR(node)) {
                    // In this case, Data WRITE lop goes into MR, and
                    // we don't have to do anything here
                    doRmVar = false;
                } else {
                    out = setupNodeOutputs(node, ExecType.CP, false, false);
                    if (dnode.getDataType() == DataType.SCALAR) {
                        // processing is same for both transient and persistent scalar writes
                        writeInst.addAll(out.getLastInstructions());
                        doRmVar = false;
                    } else {
                        // setupNodeOutputs() handles both transient and persistent matrix writes
                        if (dnode.isTransient()) {
                            deleteInst.addAll(out.getLastInstructions());
                            doRmVar = false;
                        } else {
                            // In case of persistent write lop, write instruction will be generated
                            // and that instruction must be added to <code>inst</code> so that it gets
                            // executed immediately. If it is added to <code>deleteInst</code> then it
                            // gets executed at the end of program block's execution
                            inst.addAll(out.getLastInstructions());
                            doRmVar = true;
                        }
                    }
                    markedNodes.add(node);
                }
            } else {
                // generate a temp label to hold the value that is read from HDFS
                if (node.getDataType() == DataType.SCALAR) {
                    node.getOutputParameters().setLabel(Lop.SCALAR_VAR_NAME_PREFIX + var_index.getNextID());
                    String io_inst = node.getInstructions(node.getOutputParameters().getLabel(), node.getOutputParameters().getFile_name());
                    CPInstruction currInstr = CPInstructionParser.parseSingleInstruction(io_inst);
                    currInstr.setLocation(node);
                    inst.add(currInstr);
                    Instruction tempInstr = VariableCPInstruction.prepareRemoveInstruction(node.getOutputParameters().getLabel());
                    tempInstr.setLocation(node);
                    deleteInst.add(tempInstr);
                } else {
                    throw new LopsException("Matrix READs are not handled in CP yet!");
                }
                markedNodes.add(node);
                doRmVar = true;
            }
        }
        // see if rmvar instructions can be generated for node's inputs
        if (doRmVar)
            processConsumersForInputs(node, inst, deleteInst);
        doRmVar = false;
    }
    for (String var : var_deletions) {
        Instruction rmInst = VariableCPInstruction.prepareRemoveInstruction(var);
        if (LOG.isTraceEnabled())
            LOG.trace("  Adding var_deletions: " + rmInst.toString());
        rmInst.setLocation(var_deletionsLineNum.get(var));
        deleteInst.add(rmInst);
    }
    // delete all marked nodes
    for (Lop node : markedNodes) {
        execNodes.remove(node);
    }
}

Example 8 with CPInstruction

use of org.apache.sysml.runtime.instructions.cp.CPInstruction in project systemml by apache.

the class Dag method generateControlProgramJobs.

/**
 * Method to generate instructions that are executed in Control Program. At
 * this point, this DAG has no dependencies on the MR dag. ie. none of the
 * inputs are outputs of MR jobs
 *
 * @param execNodes list of low-level operators
 * @param inst list of instructions
 * @param writeInst list of write instructions
 * @param deleteInst list of delete instructions
 */
private void generateControlProgramJobs(ArrayList<Lop> execNodes, ArrayList<Instruction> inst, ArrayList<Instruction> writeInst, ArrayList<Instruction> deleteInst) {
    // nodes to be deleted from execnodes
    ArrayList<Lop> markedNodes = new ArrayList<>();
    // variable names to be deleted
    ArrayList<String> var_deletions = new ArrayList<>();
    HashMap<String, Lop> var_deletionsLineNum = new HashMap<>();
    boolean doRmVar = false;
    for (int i = 0; i < execNodes.size(); i++) {
        Lop node = execNodes.get(i);
        doRmVar = false;
        // TODO: statiko -- check if this condition ever evaluated to TRUE
        if (node.getExecLocation() == ExecLocation.Data && ((Data) node).getOperationType() == Data.OperationTypes.READ && ((Data) node).getDataType() == DataType.SCALAR && node.getOutputParameters().getFile_name() == null) {
            markedNodes.add(node);
            continue;
        }
        // output scalar instructions and mark nodes for deletion
        if (node.getExecLocation() == ExecLocation.ControlProgram) {
            if (node.getDataType() == DataType.SCALAR) {
                // Output from lops with SCALAR data type must
                // go into Temporary Variables (Var0, Var1, etc.)
                NodeOutput out = setupNodeOutputs(node, ExecType.CP, false, false);
                // dummy
                inst.addAll(out.getPreInstructions());
                deleteInst.addAll(out.getLastInstructions());
            } else {
                // Output from lops with non-SCALAR data type must
                // go into Temporary Files (temp0, temp1, etc.)
                NodeOutput out = setupNodeOutputs(node, ExecType.CP, false, false);
                inst.addAll(out.getPreInstructions());
                boolean hasTransientWriteParent = false;
                for (Lop parent : node.getOutputs()) {
                    if (parent.getExecLocation() == ExecLocation.Data && ((Data) parent).getOperationType() == Data.OperationTypes.WRITE && ((Data) parent).isTransient()) {
                        hasTransientWriteParent = true;
                        break;
                    }
                }
                if (!hasTransientWriteParent) {
                    deleteInst.addAll(out.getLastInstructions());
                } else {
                    var_deletions.add(node.getOutputParameters().getLabel());
                    var_deletionsLineNum.put(node.getOutputParameters().getLabel(), node);
                }
            }
            String inst_string = "";
            // are handled separately, by simply passing ONLY the output variable to getInstructions()
            if (node.getType() == Lop.Type.ParameterizedBuiltin || node.getType() == Lop.Type.GroupedAgg || node.getType() == Lop.Type.DataGen) {
                inst_string = node.getInstructions(node.getOutputParameters().getLabel());
            } else // separately as well by passing arrays of inputs and outputs
            if (node.getType() == Lop.Type.FunctionCallCP) {
                String[] inputs = new String[node.getInputs().size()];
                String[] outputs = new String[node.getOutputs().size()];
                int count = 0;
                for (Lop in : node.getInputs()) inputs[count++] = in.getOutputParameters().getLabel();
                count = 0;
                for (Lop out : node.getOutputs()) outputs[count++] = out.getOutputParameters().getLabel();
                inst_string = node.getInstructions(inputs, outputs);
            } else if (node.getType() == Lop.Type.Nary) {
                String[] inputs = new String[node.getInputs().size()];
                int count = 0;
                for (Lop in : node.getInputs()) inputs[count++] = in.getOutputParameters().getLabel();
                inst_string = node.getInstructions(inputs, node.getOutputParameters().getLabel());
            } else {
                if (node.getInputs().isEmpty()) {
                    // currently, such a case exists only for Rand lop
                    inst_string = node.getInstructions(node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 1) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 2) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 3 || node.getType() == Type.Ctable) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 4) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getInputs().get(3).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 5) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getInputs().get(3).getOutputParameters().getLabel(), node.getInputs().get(4).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 6) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getInputs().get(3).getOutputParameters().getLabel(), node.getInputs().get(4).getOutputParameters().getLabel(), node.getInputs().get(5).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 7) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getInputs().get(3).getOutputParameters().getLabel(), node.getInputs().get(4).getOutputParameters().getLabel(), node.getInputs().get(5).getOutputParameters().getLabel(), node.getInputs().get(6).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else {
                    String[] inputs = new String[node.getInputs().size()];
                    for (int j = 0; j < node.getInputs().size(); j++) inputs[j] = node.getInputs().get(j).getOutputParameters().getLabel();
                    inst_string = node.getInstructions(inputs, node.getOutputParameters().getLabel());
                }
            }
            try {
                if (LOG.isTraceEnabled())
                    LOG.trace("Generating instruction - " + inst_string);
                Instruction currInstr = InstructionParser.parseSingleInstruction(inst_string);
                if (currInstr == null) {
                    throw new LopsException("Error parsing the instruction:" + inst_string);
                }
                if (node._beginLine != 0)
                    currInstr.setLocation(node);
                else if (!node.getOutputs().isEmpty())
                    currInstr.setLocation(node.getOutputs().get(0));
                else if (!node.getInputs().isEmpty())
                    currInstr.setLocation(node.getInputs().get(0));
                inst.add(currInstr);
            } catch (Exception e) {
                throw new LopsException(node.printErrorLocation() + "Problem generating simple inst - " + inst_string, e);
            }
            markedNodes.add(node);
            doRmVar = true;
        } else if (node.getExecLocation() == ExecLocation.Data) {
            Data dnode = (Data) node;
            Data.OperationTypes op = dnode.getOperationType();
            if (op == Data.OperationTypes.WRITE) {
                NodeOutput out = null;
                if (sendWriteLopToMR(node)) {
                    // In this case, Data WRITE lop goes into MR, and
                    // we don't have to do anything here
                    doRmVar = false;
                } else {
                    out = setupNodeOutputs(node, ExecType.CP, false, false);
                    if (dnode.getDataType() == DataType.SCALAR) {
                        // processing is same for both transient and persistent scalar writes
                        writeInst.addAll(out.getLastInstructions());
                        doRmVar = false;
                    } else {
                        // setupNodeOutputs() handles both transient and persistent matrix writes
                        if (dnode.isTransient()) {
                            deleteInst.addAll(out.getLastInstructions());
                            doRmVar = false;
                        } else {
                            // In case of persistent write lop, write instruction will be generated
                            // and that instruction must be added to <code>inst</code> so that it gets
                            // executed immediately. If it is added to <code>deleteInst</code> then it
                            // gets executed at the end of program block's execution
                            inst.addAll(out.getLastInstructions());
                            doRmVar = true;
                        }
                    }
                    markedNodes.add(node);
                }
            } else {
                // generate a temp label to hold the value that is read from HDFS
                if (node.getDataType() == DataType.SCALAR) {
                    node.getOutputParameters().setLabel(Lop.SCALAR_VAR_NAME_PREFIX + var_index.getNextID());
                    String io_inst = node.getInstructions(node.getOutputParameters().getLabel(), node.getOutputParameters().getFile_name());
                    CPInstruction currInstr = CPInstructionParser.parseSingleInstruction(io_inst);
                    currInstr.setLocation(node);
                    inst.add(currInstr);
                    Instruction tempInstr = VariableCPInstruction.prepareRemoveInstruction(node.getOutputParameters().getLabel());
                    tempInstr.setLocation(node);
                    deleteInst.add(tempInstr);
                } else {
                    throw new LopsException("Matrix READs are not handled in CP yet!");
                }
                markedNodes.add(node);
                doRmVar = true;
            }
        }
        // see if rmvar instructions can be generated for node's inputs
        if (doRmVar)
            processConsumersForInputs(node, inst, deleteInst);
        doRmVar = false;
    }
    for (String var : var_deletions) {
        Instruction rmInst = VariableCPInstruction.prepareRemoveInstruction(var);
        if (LOG.isTraceEnabled())
            LOG.trace("  Adding var_deletions: " + rmInst.toString());
        rmInst.setLocation(var_deletionsLineNum.get(var));
        deleteInst.add(rmInst);
    }
    // delete all marked nodes
    for (Lop node : markedNodes) {
        execNodes.remove(node);
    }
}

Example 9 with CPInstruction

use of org.apache.sysml.runtime.instructions.cp.CPInstruction in project systemml by apache.

the class CostEstimator method rGetTimeEstimate.

private double rGetTimeEstimate(ProgramBlock pb, HashMap<String, VarStats> stats, HashSet<String> memoFunc, boolean recursive) {
    double ret = 0;
    if (pb instanceof WhileProgramBlock) {
        WhileProgramBlock tmp = (WhileProgramBlock) pb;
        if (recursive)
            for (ProgramBlock pb2 : tmp.getChildBlocks()) ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
        ret *= DEFAULT_NUMITER;
    } else if (pb instanceof IfProgramBlock) {
        IfProgramBlock tmp = (IfProgramBlock) pb;
        if (recursive) {
            for (ProgramBlock pb2 : tmp.getChildBlocksIfBody()) ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
            if (tmp.getChildBlocksElseBody() != null)
                for (ProgramBlock pb2 : tmp.getChildBlocksElseBody()) {
                    ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
                    // weighted sum
                    ret /= 2;
                }
        }
    } else if (// includes ParFORProgramBlock
    pb instanceof ForProgramBlock) {
        ForProgramBlock tmp = (ForProgramBlock) pb;
        if (recursive)
            for (ProgramBlock pb2 : tmp.getChildBlocks()) ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
        ret *= getNumIterations(stats, tmp);
    } else if (pb instanceof FunctionProgramBlock && // see generic
    !(pb instanceof ExternalFunctionProgramBlock)) {
        FunctionProgramBlock tmp = (FunctionProgramBlock) pb;
        if (recursive)
            for (ProgramBlock pb2 : tmp.getChildBlocks()) ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
    } else {
        ArrayList<Instruction> tmp = pb.getInstructions();
        for (Instruction inst : tmp) {
            if (// CP
            inst instanceof CPInstruction) {
                // obtain stats from createvar, cpvar, rmvar, rand
                maintainCPInstVariableStatistics((CPInstruction) inst, stats);
                // extract statistics (instruction-specific)
                Object[] o = extractCPInstStatistics(inst, stats);
                VarStats[] vs = (VarStats[]) o[0];
                String[] attr = (String[]) o[1];
                // if(LOG.isDebugEnabled())
                // LOG.debug(inst);
                // call time estimation for inst
                ret += getCPInstTimeEstimate(inst, vs, attr);
                if (// functions
                inst instanceof FunctionCallCPInstruction) {
                    FunctionCallCPInstruction finst = (FunctionCallCPInstruction) inst;
                    String fkey = DMLProgram.constructFunctionKey(finst.getNamespace(), finst.getFunctionName());
                    // awareness of recursive functions, missing program
                    if (!memoFunc.contains(fkey) && pb.getProgram() != null) {
                        if (LOG.isDebugEnabled())
                            LOG.debug("Begin Function " + fkey);
                        memoFunc.add(fkey);
                        Program prog = pb.getProgram();
                        FunctionProgramBlock fpb = prog.getFunctionProgramBlock(finst.getNamespace(), finst.getFunctionName());
                        ret += rGetTimeEstimate(fpb, stats, memoFunc, recursive);
                        memoFunc.remove(fkey);
                        if (LOG.isDebugEnabled())
                            LOG.debug("End Function " + fkey);
                    }
                }
            } else if (// MR
            inst instanceof MRJobInstruction) {
                // obtain stats for job
                maintainMRJobInstVariableStatistics(inst, stats);
                // extract input statistics
                Object[] o = extractMRJobInstStatistics(inst, stats);
                VarStats[] vs = (VarStats[]) o[0];
                if (LOG.isDebugEnabled())
                    LOG.debug("Begin MRJob type=" + ((MRJobInstruction) inst).getJobType());
                // call time estimation for complex MR inst
                ret += getMRJobInstTimeEstimate(inst, vs, null);
                if (LOG.isDebugEnabled())
                    LOG.debug("End MRJob");
                // cleanup stats for job
                cleanupMRJobVariableStatistics(inst, stats);
            }
        }
    }
    return ret;
}

Example 10 with CPInstruction

use of org.apache.sysml.runtime.instructions.cp.CPInstruction in project systemml by apache.

the class CostEstimatorStaticRuntime method getCPInstTimeEstimate.

@Override
@SuppressWarnings("unused")
protected double getCPInstTimeEstimate(Instruction inst, VarStats[] vs, String[] args) {
    CPInstruction cpinst = (CPInstruction) inst;
    // load time into mem
    double ltime = 0;
    if (!vs[0]._inmem) {
        ltime += getHDFSReadTime(vs[0]._rlen, vs[0]._clen, vs[0].getSparsity());
        // eviction costs
        if (CacheableData.CACHING_WRITE_CACHE_ON_READ && LazyWriteBuffer.getWriteBufferLimit() < MatrixBlock.estimateSizeOnDisk(vs[0]._rlen, vs[0]._clen, (long) ((vs[0]._nnz < 0) ? vs[0]._rlen * vs[0]._clen : vs[0]._nnz))) {
            ltime += Math.abs(getFSWriteTime(vs[0]._rlen, vs[0]._clen, vs[0].getSparsity()));
        }
        vs[0]._inmem = true;
    }
    if (!vs[1]._inmem) {
        ltime += getHDFSReadTime(vs[1]._rlen, vs[1]._clen, vs[1].getSparsity());
        // eviction costs
        if (CacheableData.CACHING_WRITE_CACHE_ON_READ && LazyWriteBuffer.getWriteBufferLimit() < MatrixBlock.estimateSizeOnDisk(vs[1]._rlen, vs[1]._clen, (long) ((vs[1]._nnz < 0) ? vs[1]._rlen * vs[1]._clen : vs[1]._nnz))) {
            ltime += Math.abs(getFSWriteTime(vs[1]._rlen, vs[1]._clen, vs[1].getSparsity()));
        }
        vs[1]._inmem = true;
    }
    if (LOG.isDebugEnabled() && ltime != 0) {
        LOG.debug("Cost[" + cpinst.getOpcode() + " - read] = " + ltime);
    }
    // exec time CP instruction
    String opcode = (cpinst instanceof FunctionCallCPInstruction) ? InstructionUtils.getOpCode(cpinst.toString()) : cpinst.getOpcode();
    double etime = getInstTimeEstimate(opcode, vs, args, ExecType.CP);
    // write time caching
    double wtime = 0;
    // double wtime = getFSWriteTime( vs[2]._rlen, vs[2]._clen, (vs[2]._nnz<0)? 1.0:(double)vs[2]._nnz/vs[2]._rlen/vs[2]._clen );
    if (inst instanceof VariableCPInstruction && ((VariableCPInstruction) inst).getOpcode().equals("write"))
        wtime += getHDFSWriteTime(vs[2]._rlen, vs[2]._clen, vs[2].getSparsity(), ((VariableCPInstruction) inst).getInput3().getName());
    if (LOG.isDebugEnabled() && wtime != 0) {
        LOG.debug("Cost[" + cpinst.getOpcode() + " - write] = " + wtime);
    }
    // total costs
    double costs = ltime + etime + wtime;
    return costs;
}