Examples with FunctionCallCPInstruction org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction used on opensource projects

Example 6 with FunctionCallCPInstruction

use of org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction in project incubator-systemml by apache.

the class CostEstimator method rGetTimeEstimate.

private double rGetTimeEstimate(ProgramBlock pb, HashMap<String, VarStats> stats, HashSet<String> memoFunc, boolean recursive) {
    double ret = 0;
    if (pb instanceof WhileProgramBlock) {
        WhileProgramBlock tmp = (WhileProgramBlock) pb;
        if (recursive)
            for (ProgramBlock pb2 : tmp.getChildBlocks()) ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
        ret *= DEFAULT_NUMITER;
    } else if (pb instanceof IfProgramBlock) {
        IfProgramBlock tmp = (IfProgramBlock) pb;
        if (recursive) {
            for (ProgramBlock pb2 : tmp.getChildBlocksIfBody()) ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
            if (tmp.getChildBlocksElseBody() != null)
                for (ProgramBlock pb2 : tmp.getChildBlocksElseBody()) {
                    ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
                    // weighted sum
                    ret /= 2;
                }
        }
    } else if (// includes ParFORProgramBlock
    pb instanceof ForProgramBlock) {
        ForProgramBlock tmp = (ForProgramBlock) pb;
        if (recursive)
            for (ProgramBlock pb2 : tmp.getChildBlocks()) ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
        ret *= getNumIterations(stats, tmp);
    } else if (pb instanceof FunctionProgramBlock && // see generic
    !(pb instanceof ExternalFunctionProgramBlock)) {
        FunctionProgramBlock tmp = (FunctionProgramBlock) pb;
        if (recursive)
            for (ProgramBlock pb2 : tmp.getChildBlocks()) ret += rGetTimeEstimate(pb2, stats, memoFunc, recursive);
    } else {
        ArrayList<Instruction> tmp = pb.getInstructions();
        for (Instruction inst : tmp) {
            if (// CP
            inst instanceof CPInstruction) {
                // obtain stats from createvar, cpvar, rmvar, rand
                maintainCPInstVariableStatistics((CPInstruction) inst, stats);
                // extract statistics (instruction-specific)
                Object[] o = extractCPInstStatistics(inst, stats);
                VarStats[] vs = (VarStats[]) o[0];
                String[] attr = (String[]) o[1];
                // if(LOG.isDebugEnabled())
                // LOG.debug(inst);
                // call time estimation for inst
                ret += getCPInstTimeEstimate(inst, vs, attr);
                if (// functions
                inst instanceof FunctionCallCPInstruction) {
                    FunctionCallCPInstruction finst = (FunctionCallCPInstruction) inst;
                    String fkey = DMLProgram.constructFunctionKey(finst.getNamespace(), finst.getFunctionName());
                    // awareness of recursive functions, missing program
                    if (!memoFunc.contains(fkey) && pb.getProgram() != null) {
                        if (LOG.isDebugEnabled())
                            LOG.debug("Begin Function " + fkey);
                        memoFunc.add(fkey);
                        Program prog = pb.getProgram();
                        FunctionProgramBlock fpb = prog.getFunctionProgramBlock(finst.getNamespace(), finst.getFunctionName());
                        ret += rGetTimeEstimate(fpb, stats, memoFunc, recursive);
                        memoFunc.remove(fkey);
                        if (LOG.isDebugEnabled())
                            LOG.debug("End Function " + fkey);
                    }
                }
            } else if (// MR
            inst instanceof MRJobInstruction) {
                // obtain stats for job
                maintainMRJobInstVariableStatistics(inst, stats);
                // extract input statistics
                Object[] o = extractMRJobInstStatistics(inst, stats);
                VarStats[] vs = (VarStats[]) o[0];
                if (LOG.isDebugEnabled())
                    LOG.debug("Begin MRJob type=" + ((MRJobInstruction) inst).getJobType());
                // call time estimation for complex MR inst
                ret += getMRJobInstTimeEstimate(inst, vs, null);
                if (LOG.isDebugEnabled())
                    LOG.debug("End MRJob");
                // cleanup stats for job
                cleanupMRJobVariableStatistics(inst, stats);
            }
        }
    }
    return ret;
}

Example 7 with FunctionCallCPInstruction

use of org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction in project incubator-systemml by apache.

the class CostEstimator method maintainCPInstVariableStatistics.

private static void maintainCPInstVariableStatistics(CPInstruction inst, HashMap<String, VarStats> stats) {
    if (inst instanceof VariableCPInstruction) {
        String optype = inst.getOpcode();
        String[] parts = InstructionUtils.getInstructionParts(inst.toString());
        if (optype.equals("createvar")) {
            if (parts.length < 10)
                return;
            String varname = parts[1];
            long rlen = Long.parseLong(parts[6]);
            long clen = Long.parseLong(parts[7]);
            int brlen = Integer.parseInt(parts[8]);
            int bclen = Integer.parseInt(parts[9]);
            long nnz = Long.parseLong(parts[10]);
            VarStats vs = new VarStats(rlen, clen, brlen, bclen, nnz, false);
            stats.put(varname, vs);
        } else if (optype.equals("cpvar")) {
            String varname = parts[1];
            String varname2 = parts[2];
            VarStats vs = stats.get(varname);
            stats.put(varname2, vs);
        } else if (optype.equals("mvvar")) {
            String varname = parts[1];
            String varname2 = parts[2];
            VarStats vs = stats.remove(varname);
            stats.put(varname2, vs);
        } else if (optype.equals("rmvar")) {
            String varname = parts[1];
            stats.remove(varname);
        }
    } else if (inst instanceof DataGenCPInstruction) {
        DataGenCPInstruction randInst = (DataGenCPInstruction) inst;
        String varname = randInst.output.getName();
        long rlen = randInst.getRows();
        long clen = randInst.getCols();
        int brlen = randInst.getRowsInBlock();
        int bclen = randInst.getColsInBlock();
        long nnz = (long) (randInst.getSparsity() * rlen * clen);
        VarStats vs = new VarStats(rlen, clen, brlen, bclen, nnz, true);
        stats.put(varname, vs);
    } else if (inst instanceof StringInitCPInstruction) {
        StringInitCPInstruction iinst = (StringInitCPInstruction) inst;
        String varname = iinst.output.getName();
        long rlen = iinst.getRows();
        long clen = iinst.getCols();
        VarStats vs = new VarStats(rlen, clen, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), rlen * clen, true);
        stats.put(varname, vs);
    } else if (inst instanceof FunctionCallCPInstruction) {
        FunctionCallCPInstruction finst = (FunctionCallCPInstruction) inst;
        ArrayList<String> outVars = finst.getBoundOutputParamNames();
        for (String varname : outVars) stats.put(varname, _unknownStats);
    }
}

Example 8 with FunctionCallCPInstruction

use of org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction in project incubator-systemml by apache.

the class CostEstimatorStaticRuntime method getCPInstTimeEstimate.

@Override
@SuppressWarnings("unused")
protected double getCPInstTimeEstimate(Instruction inst, VarStats[] vs, String[] args) {
    CPInstruction cpinst = (CPInstruction) inst;
    // load time into mem
    double ltime = 0;
    if (!vs[0]._inmem) {
        ltime += getHDFSReadTime(vs[0]._rlen, vs[0]._clen, vs[0].getSparsity());
        // eviction costs
        if (CacheableData.CACHING_WRITE_CACHE_ON_READ && LazyWriteBuffer.getWriteBufferLimit() < MatrixBlock.estimateSizeOnDisk(vs[0]._rlen, vs[0]._clen, (long) ((vs[0]._nnz < 0) ? vs[0]._rlen * vs[0]._clen : vs[0]._nnz))) {
            ltime += Math.abs(getFSWriteTime(vs[0]._rlen, vs[0]._clen, vs[0].getSparsity()));
        }
        vs[0]._inmem = true;
    }
    if (!vs[1]._inmem) {
        ltime += getHDFSReadTime(vs[1]._rlen, vs[1]._clen, vs[1].getSparsity());
        // eviction costs
        if (CacheableData.CACHING_WRITE_CACHE_ON_READ && LazyWriteBuffer.getWriteBufferLimit() < MatrixBlock.estimateSizeOnDisk(vs[1]._rlen, vs[1]._clen, (long) ((vs[1]._nnz < 0) ? vs[1]._rlen * vs[1]._clen : vs[1]._nnz))) {
            ltime += Math.abs(getFSWriteTime(vs[1]._rlen, vs[1]._clen, vs[1].getSparsity()));
        }
        vs[1]._inmem = true;
    }
    if (LOG.isDebugEnabled() && ltime != 0) {
        LOG.debug("Cost[" + cpinst.getOpcode() + " - read] = " + ltime);
    }
    // exec time CP instruction
    String opcode = (cpinst instanceof FunctionCallCPInstruction) ? InstructionUtils.getOpCode(cpinst.toString()) : cpinst.getOpcode();
    double etime = getInstTimeEstimate(opcode, vs, args, ExecType.CP);
    // write time caching
    double wtime = 0;
    // double wtime = getFSWriteTime( vs[2]._rlen, vs[2]._clen, (vs[2]._nnz<0)? 1.0:(double)vs[2]._nnz/vs[2]._rlen/vs[2]._clen );
    if (inst instanceof VariableCPInstruction && ((VariableCPInstruction) inst).getOpcode().equals("write"))
        wtime += getHDFSWriteTime(vs[2]._rlen, vs[2]._clen, vs[2].getSparsity(), ((VariableCPInstruction) inst).getInput3().getName());
    if (LOG.isDebugEnabled() && wtime != 0) {
        LOG.debug("Cost[" + cpinst.getOpcode() + " - write] = " + wtime);
    }
    // total costs
    double costs = ltime + etime + wtime;
    return costs;
}

Example 9 with FunctionCallCPInstruction

use of org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction in project incubator-systemml by apache.

the class Recompiler method rRecompileProgramBlock2Forced.

private static void rRecompileProgramBlock2Forced(ProgramBlock pb, long tid, HashSet<String> fnStack, ExecType et) {
    if (pb instanceof WhileProgramBlock) {
        WhileProgramBlock pbTmp = (WhileProgramBlock) pb;
        WhileStatementBlock sbTmp = (WhileStatementBlock) pbTmp.getStatementBlock();
        // recompile predicate
        if (sbTmp != null && !(et == ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getPredicate(), true, true)))
            pbTmp.setPredicate(Recompiler.recompileHopsDag2Forced(sbTmp.getPredicateHops(), tid, et));
        // recompile body
        for (ProgramBlock pb2 : pbTmp.getChildBlocks()) rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
    } else if (pb instanceof IfProgramBlock) {
        IfProgramBlock pbTmp = (IfProgramBlock) pb;
        IfStatementBlock sbTmp = (IfStatementBlock) pbTmp.getStatementBlock();
        // recompile predicate
        if (sbTmp != null && !(et == ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getPredicate(), true, true)))
            pbTmp.setPredicate(Recompiler.recompileHopsDag2Forced(sbTmp.getPredicateHops(), tid, et));
        // recompile body
        for (ProgramBlock pb2 : pbTmp.getChildBlocksIfBody()) rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
        for (ProgramBlock pb2 : pbTmp.getChildBlocksElseBody()) rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
    } else if (// includes ParFORProgramBlock
    pb instanceof ForProgramBlock) {
        ForProgramBlock pbTmp = (ForProgramBlock) pb;
        ForStatementBlock sbTmp = (ForStatementBlock) pbTmp.getStatementBlock();
        // recompile predicate
        if (sbTmp != null && sbTmp.getFromHops() != null && !(et == ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getFromInstructions(), true, true)))
            pbTmp.setFromInstructions(Recompiler.recompileHopsDag2Forced(sbTmp.getFromHops(), tid, et));
        if (sbTmp != null && sbTmp.getToHops() != null && !(et == ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getToInstructions(), true, true)))
            pbTmp.setToInstructions(Recompiler.recompileHopsDag2Forced(sbTmp.getToHops(), tid, et));
        if (sbTmp != null && sbTmp.getIncrementHops() != null && !(et == ExecType.CP && !OptTreeConverter.containsMRJobInstruction(pbTmp.getIncrementInstructions(), true, true)))
            pbTmp.setIncrementInstructions(Recompiler.recompileHopsDag2Forced(sbTmp.getIncrementHops(), tid, et));
        // recompile body
        for (ProgramBlock pb2 : pbTmp.getChildBlocks()) rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
    } else if (// includes ExternalFunctionProgramBlock and ExternalFunctionProgramBlockCP
    pb instanceof FunctionProgramBlock) {
        FunctionProgramBlock tmp = (FunctionProgramBlock) pb;
        for (ProgramBlock pb2 : tmp.getChildBlocks()) rRecompileProgramBlock2Forced(pb2, tid, fnStack, et);
    } else {
        StatementBlock sb = pb.getStatementBlock();
        // would be invalid with permutation matrix mult across multiple dags)
        if (sb != null) {
            ArrayList<Instruction> tmp = pb.getInstructions();
            tmp = Recompiler.recompileHopsDag2Forced(sb, sb.getHops(), tid, et);
            pb.setInstructions(tmp);
        }
        // recompile functions
        if (OptTreeConverter.containsFunctionCallInstruction(pb)) {
            ArrayList<Instruction> tmp = pb.getInstructions();
            for (Instruction inst : tmp) if (inst instanceof FunctionCallCPInstruction) {
                FunctionCallCPInstruction func = (FunctionCallCPInstruction) inst;
                String fname = func.getFunctionName();
                String fnamespace = func.getNamespace();
                String fKey = DMLProgram.constructFunctionKey(fnamespace, fname);
                if (// memoization for multiple calls, recursion
                !fnStack.contains(fKey)) {
                    fnStack.add(fKey);
                    FunctionProgramBlock fpb = pb.getProgram().getFunctionProgramBlock(fnamespace, fname);
                    // recompile chains of functions
                    rRecompileProgramBlock2Forced(fpb, tid, fnStack, et);
                }
            }
        }
    }
}

Example 10 with FunctionCallCPInstruction

use of org.apache.sysml.runtime.instructions.cp.FunctionCallCPInstruction in project incubator-systemml by apache.

the class ExecutionContext method suspendIfAskedInDebugMode.

/**
 * This function should be called only if user has specified -debug option.
 * In this function, if the user has issued one of the step instructions or
 * has enabled suspend flag in previous instruction (through breakpoint),
 * then it will wait until user issues a new debugger command.
 *
 * @param currInst current instruction
 */
@SuppressWarnings("deprecation")
private void suspendIfAskedInDebugMode(Instruction currInst) {
    if (!DMLScript.ENABLE_DEBUG_MODE) {
        System.err.println("ERROR: The function suspendIfAskedInDebugMode should not be called in non-debug mode.");
    }
    // check for stepping options
    if (!_dbState.suspend && _dbState.dbCommand != null) {
        if (_dbState.dbCommand.equalsIgnoreCase("step_instruction")) {
            System.out.format("Step instruction reached at %s.\n", _dbState.getPC().toString());
            _dbState.suspend = true;
        } else if (_dbState.dbCommand.equalsIgnoreCase("step_line") && _dbState.prevPC.getLineNumber() != currInst.getLineNum() && _dbState.prevPC.getLineNumber() != 0) {
            // Don't step into first instruction of first line
            // System.out.format("Step reached at %s.\n", this._prog.getPC().toString());
            System.out.format("Step reached at %s.\n", _dbState.getPC().toStringWithoutInstructionID());
            _dbState.suspend = true;
        } else if (_dbState.dbCommand.equalsIgnoreCase("step return") && currInst instanceof FunctionCallCPInstruction) {
            FunctionCallCPInstruction funCallInst = (FunctionCallCPInstruction) currInst;
            if (_dbState.dbCommandArg == null || funCallInst.getFunctionName().equalsIgnoreCase(_dbState.dbCommandArg)) {
                System.out.format("Step return reached at %s.\n", _dbState.getPC().toStringWithoutInstructionID());
                _dbState.suspend = true;
            }
        }
    }
    // check if runtime suspend signal is set
    if (_dbState.suspend) {
        // flush old commands and arguments
        _dbState.dbCommand = null;
        _dbState.dbCommandArg = null;
        // print current DML script source line
        if (currInst.getLineNum() != 0)
            _dbState.printDMLSourceLine(currInst.getLineNum());
        // save current symbol table
        _dbState.setVariables(this.getVariables());
        // send next command signal to debugger control module
        _dbState.nextCommand = true;
        // suspend runtime execution thread
        Thread.currentThread().suspend();
        // reset next command signal
        _dbState.nextCommand = false;
    }
    // reset runtime suspend signal
    _dbState.suspend = false;
    // update previous pc
    _dbState.prevPC.setFunctionName(_dbState.getPC().getFunctionName());
    _dbState.prevPC.setProgramBlockNumber(_dbState.getPC().getProgramBlockNumber());
    _dbState.prevPC.setLineNumber(currInst.getLineNum());
}