Examples with VariableCPInstruction org.apache.sysml.runtime.instructions.cp.VariableCPInstruction used on opensource projects

Example 1 with VariableCPInstruction

use of org.apache.sysml.runtime.instructions.cp.VariableCPInstruction in project incubator-systemml by apache.

the class CostEstimatorStaticRuntime method getCPInstTimeEstimate.

@Override
@SuppressWarnings("unused")
protected double getCPInstTimeEstimate(Instruction inst, VarStats[] vs, String[] args) throws DMLRuntimeException {
    CPInstruction cpinst = (CPInstruction) inst;
    //load time into mem
    double ltime = 0;
    if (!vs[0]._inmem) {
        ltime += getHDFSReadTime(vs[0]._rlen, vs[0]._clen, vs[0].getSparsity());
        //eviction costs
        if (CacheableData.CACHING_WRITE_CACHE_ON_READ && LazyWriteBuffer.getWriteBufferSize() < MatrixBlock.estimateSizeOnDisk(vs[0]._rlen, vs[0]._clen, (long) ((vs[0]._nnz < 0) ? vs[0]._rlen * vs[0]._clen : vs[0]._nnz))) {
            ltime += Math.abs(getFSWriteTime(vs[0]._rlen, vs[0]._clen, vs[0].getSparsity()));
        }
        vs[0]._inmem = true;
    }
    if (!vs[1]._inmem) {
        ltime += getHDFSReadTime(vs[1]._rlen, vs[1]._clen, vs[1].getSparsity());
        //eviction costs
        if (CacheableData.CACHING_WRITE_CACHE_ON_READ && LazyWriteBuffer.getWriteBufferSize() < MatrixBlock.estimateSizeOnDisk(vs[1]._rlen, vs[1]._clen, (long) ((vs[1]._nnz < 0) ? vs[1]._rlen * vs[1]._clen : vs[1]._nnz))) {
            ltime += Math.abs(getFSWriteTime(vs[1]._rlen, vs[1]._clen, vs[1].getSparsity()));
        }
        vs[1]._inmem = true;
    }
    if (LOG.isDebugEnabled() && ltime != 0) {
        LOG.debug("Cost[" + cpinst.getOpcode() + " - read] = " + ltime);
    }
    //exec time CP instruction
    String opcode = (cpinst instanceof FunctionCallCPInstruction) ? InstructionUtils.getOpCode(cpinst.toString()) : cpinst.getOpcode();
    double etime = getInstTimeEstimate(opcode, vs, args, ExecType.CP);
    //write time caching
    double wtime = 0;
    //double wtime = getFSWriteTime( vs[2]._rlen, vs[2]._clen, (vs[2]._nnz<0)? 1.0:(double)vs[2]._nnz/vs[2]._rlen/vs[2]._clen );
    if (inst instanceof VariableCPInstruction && ((VariableCPInstruction) inst).getOpcode().equals("write"))
        wtime += getHDFSWriteTime(vs[2]._rlen, vs[2]._clen, vs[2].getSparsity(), ((VariableCPInstruction) inst).getInput3().getName());
    if (LOG.isDebugEnabled() && wtime != 0) {
        LOG.debug("Cost[" + cpinst.getOpcode() + " - write] = " + wtime);
    }
    //total costs
    double costs = ltime + etime + wtime;
    return costs;
}

Example 2 with VariableCPInstruction

use of org.apache.sysml.runtime.instructions.cp.VariableCPInstruction in project incubator-systemml by apache.

the class CostEstimator method extractCPInstStatistics.

private Object[] extractCPInstStatistics(Instruction inst, HashMap<String, VarStats> stats) {
    //stats, attrs
    Object[] ret = new Object[2];
    VarStats[] vs = new VarStats[3];
    String[] attr = null;
    if (inst instanceof UnaryCPInstruction) {
        if (inst instanceof DataGenCPInstruction) {
            DataGenCPInstruction rinst = (DataGenCPInstruction) inst;
            vs[0] = _unknownStats;
            vs[1] = _unknownStats;
            vs[2] = stats.get(rinst.output.getName());
            //prepare attributes for cost estimation
            //full rand
            int type = 2;
            if (rinst.getMinValue() == 0.0 && rinst.getMaxValue() == 0.0)
                type = 0;
            else if (rinst.getSparsity() == 1.0 && rinst.getMinValue() == rinst.getMaxValue())
                type = 1;
            attr = new String[] { String.valueOf(type) };
        } else if (inst instanceof StringInitCPInstruction) {
            StringInitCPInstruction rinst = (StringInitCPInstruction) inst;
            vs[0] = _unknownStats;
            vs[1] = _unknownStats;
            vs[2] = stats.get(rinst.output.getName());
        } else //general unary
        {
            UnaryCPInstruction uinst = (UnaryCPInstruction) inst;
            vs[0] = stats.get(uinst.input1.getName());
            vs[1] = _unknownStats;
            vs[2] = stats.get(uinst.output.getName());
            if (//scalar input, e.g., print
            vs[0] == null)
                vs[0] = _scalarStats;
            if (//scalar output
            vs[2] == null)
                vs[2] = _scalarStats;
            if (inst instanceof MMTSJCPInstruction) {
                String type = ((MMTSJCPInstruction) inst).getMMTSJType().toString();
                attr = new String[] { type };
            } else if (inst instanceof AggregateUnaryCPInstruction) {
                String[] parts = InstructionUtils.getInstructionParts(inst.toString());
                String opcode = parts[0];
                if (opcode.equals("cm"))
                    attr = new String[] { parts[parts.length - 2] };
            }
        }
    } else if (inst instanceof BinaryCPInstruction) {
        BinaryCPInstruction binst = (BinaryCPInstruction) inst;
        vs[0] = stats.get(binst.input1.getName());
        vs[1] = stats.get(binst.input2.getName());
        vs[2] = stats.get(binst.output.getName());
        if (//scalar input, 
        vs[0] == null)
            vs[0] = _scalarStats;
        if (//scalar input, 
        vs[1] == null)
            vs[1] = _scalarStats;
        if (//scalar output
        vs[2] == null)
            vs[2] = _scalarStats;
    } else if (inst instanceof AggregateTernaryCPInstruction) {
        AggregateTernaryCPInstruction binst = (AggregateTernaryCPInstruction) inst;
        //of same dimension anyway but missing third input
        vs[0] = stats.get(binst.input1.getName());
        vs[1] = stats.get(binst.input2.getName());
        vs[2] = stats.get(binst.output.getName());
        if (//scalar input, 
        vs[0] == null)
            vs[0] = _scalarStats;
        if (//scalar input, 
        vs[1] == null)
            vs[1] = _scalarStats;
        if (//scalar output
        vs[2] == null)
            vs[2] = _scalarStats;
    } else if (inst instanceof ParameterizedBuiltinCPInstruction) {
        //ParameterizedBuiltinCPInstruction pinst = (ParameterizedBuiltinCPInstruction) inst;
        String[] parts = InstructionUtils.getInstructionParts(inst.toString());
        String opcode = parts[0];
        if (opcode.equals("groupedagg")) {
            HashMap<String, String> paramsMap = ParameterizedBuiltinCPInstruction.constructParameterMap(parts);
            String fn = paramsMap.get("fn");
            String order = paramsMap.get("order");
            AggregateOperationTypes type = CMOperator.getAggOpType(fn, order);
            attr = new String[] { String.valueOf(type.ordinal()) };
        } else if (opcode.equals("rmempty")) {
            HashMap<String, String> paramsMap = ParameterizedBuiltinCPInstruction.constructParameterMap(parts);
            attr = new String[] { String.valueOf(paramsMap.get("margin").equals("rows") ? 0 : 1) };
        }
        vs[0] = stats.get(parts[1].substring(7).replaceAll(Lop.VARIABLE_NAME_PLACEHOLDER, ""));
        //TODO
        vs[1] = _unknownStats;
        vs[2] = stats.get(parts[parts.length - 1]);
        if (//scalar input
        vs[0] == null)
            vs[0] = _scalarStats;
        if (//scalar output
        vs[2] == null)
            vs[2] = _scalarStats;
    } else if (inst instanceof MultiReturnBuiltinCPInstruction) {
        //applies to qr, lu, eigen (cost computation on input1)
        MultiReturnBuiltinCPInstruction minst = (MultiReturnBuiltinCPInstruction) inst;
        vs[0] = stats.get(minst.input1.getName());
        vs[1] = stats.get(minst.getOutput(0).getName());
        vs[2] = stats.get(minst.getOutput(1).getName());
    } else if (inst instanceof VariableCPInstruction) {
        setUnknownStats(vs);
        VariableCPInstruction varinst = (VariableCPInstruction) inst;
        if (varinst.getOpcode().equals("write")) {
            //special handling write of matrix objects (non existing if scalar)
            if (stats.containsKey(varinst.getInput1().getName()))
                vs[0] = stats.get(varinst.getInput1().getName());
            attr = new String[] { varinst.getInput3().getName() };
        }
    } else {
        setUnknownStats(vs);
    }
    //maintain var status (CP output always inmem)
    vs[2]._inmem = true;
    ret[0] = vs;
    ret[1] = attr;
    return ret;
}

Example 3 with VariableCPInstruction

use of org.apache.sysml.runtime.instructions.cp.VariableCPInstruction in project incubator-systemml by apache.

the class CostEstimator method maintainCPInstVariableStatistics.

private void maintainCPInstVariableStatistics(CPInstruction inst, HashMap<String, VarStats> stats) {
    if (inst instanceof VariableCPInstruction) {
        String optype = inst.getOpcode();
        String[] parts = InstructionUtils.getInstructionParts(inst.toString());
        if (optype.equals("createvar")) {
            if (parts.length < 10)
                return;
            String varname = parts[1];
            long rlen = Long.parseLong(parts[6]);
            long clen = Long.parseLong(parts[7]);
            long brlen = Long.parseLong(parts[8]);
            long bclen = Long.parseLong(parts[9]);
            long nnz = Long.parseLong(parts[10]);
            VarStats vs = new VarStats(rlen, clen, brlen, bclen, nnz, false);
            stats.put(varname, vs);
        //System.out.println(varname+" "+vs);
        } else if (optype.equals("cpvar")) {
            String varname = parts[1];
            String varname2 = parts[2];
            VarStats vs = stats.get(varname);
            stats.put(varname2, vs);
        } else if (optype.equals("mvvar")) {
            String varname = parts[1];
            String varname2 = parts[2];
            VarStats vs = stats.remove(varname);
            stats.put(varname2, vs);
        } else if (optype.equals("rmvar")) {
            String varname = parts[1];
            stats.remove(varname);
        }
    } else if (inst instanceof DataGenCPInstruction) {
        DataGenCPInstruction randInst = (DataGenCPInstruction) inst;
        String varname = randInst.output.getName();
        long rlen = randInst.getRows();
        long clen = randInst.getCols();
        long brlen = randInst.getRowsInBlock();
        long bclen = randInst.getColsInBlock();
        long nnz = (long) (randInst.getSparsity() * rlen * clen);
        VarStats vs = new VarStats(rlen, clen, brlen, bclen, nnz, true);
        stats.put(varname, vs);
    } else if (inst instanceof StringInitCPInstruction) {
        StringInitCPInstruction iinst = (StringInitCPInstruction) inst;
        String varname = iinst.output.getName();
        long rlen = iinst.getRows();
        long clen = iinst.getCols();
        VarStats vs = new VarStats(rlen, clen, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), rlen * clen, true);
        stats.put(varname, vs);
    } else if (inst instanceof FunctionCallCPInstruction) {
        FunctionCallCPInstruction finst = (FunctionCallCPInstruction) inst;
        ArrayList<String> outVars = finst.getBoundOutputParamNames();
        for (String varname : outVars) {
            stats.put(varname, _unknownStats);
        //System.out.println(varname+" "+vs);
        }
    }
}

Example 4 with VariableCPInstruction

use of org.apache.sysml.runtime.instructions.cp.VariableCPInstruction in project incubator-systemml by apache.

the class ProgramConverter method saveReplaceThreadID.

//////////
// CUSTOM SAFE LITERAL REPLACEMENT
/**
	 * In-place replacement of thread ids in filenames, functions names etc
	 * 
	 * @param inst instruction
	 * @param pattern ?
	 * @param replacement string replacement
	 * @return instruction
	 * @throws DMLRuntimeException if DMLRuntimeException occurs
	 */
private static Instruction saveReplaceThreadID(Instruction inst, String pattern, String replacement) throws DMLRuntimeException {
    //currently known, relevant instructions: createvar, rand, seq, extfunct, 
    if (inst instanceof MRJobInstruction) {
        //update dims file, and internal string representations of rand/seq instructions
        MRJobInstruction mrinst = (MRJobInstruction) inst;
        mrinst.updateInstructionThreadID(pattern, replacement);
    } else if (//createvar, setfilename
    inst instanceof VariableCPInstruction) {
        //update in-memory representation
        inst.updateInstructionThreadID(pattern, replacement);
    }
    return inst;
}

Example 5 with VariableCPInstruction

use of org.apache.sysml.runtime.instructions.cp.VariableCPInstruction in project incubator-systemml by apache.

the class MLContextUtil method deleteRemoveVariableInstructions.

/**
	 * Delete 'remove variable' instructions.
	 *
	 * @param instructions
	 *            list of instructions
	 */
private static void deleteRemoveVariableInstructions(ArrayList<Instruction> instructions) {
    for (int i = 0; i < instructions.size(); i++) {
        Instruction linst = instructions.get(i);
        if (linst instanceof VariableCPInstruction && ((VariableCPInstruction) linst).isRemoveVariable()) {
            VariableCPInstruction varinst = (VariableCPInstruction) linst;
            instructions.remove(varinst);
            i--;
        }
    }
}