Examples with MRInstruction org.apache.sysml.runtime.instructions.mr.MRInstruction used on opensource projects

Example 1 with MRInstruction

use of org.apache.sysml.runtime.instructions.mr.MRInstruction in project incubator-systemml by apache.

the class CostEstimatorStaticRuntime method extractMRInstStatistics.

private Object[] extractMRInstStatistics(String inst, VarStats[] stats) throws DMLRuntimeException {
    //stats, attrs
    Object[] ret = new Object[2];
    VarStats[] vs = new VarStats[3];
    String[] attr = null;
    String[] parts = InstructionUtils.getInstructionParts(inst);
    String opcode = parts[0];
    if (opcode.equals(DataGen.RAND_OPCODE)) {
        vs[0] = _unknownStats;
        vs[1] = _unknownStats;
        vs[2] = stats[Integer.parseInt(parts[2])];
        int type = 2;
        //awareness of instruction patching min/max
        if (!parts[7].contains(Lop.VARIABLE_NAME_PLACEHOLDER) && !parts[8].contains(Lop.VARIABLE_NAME_PLACEHOLDER)) {
            double minValue = Double.parseDouble(parts[7]);
            double maxValue = Double.parseDouble(parts[8]);
            double sparsity = Double.parseDouble(parts[9]);
            if (minValue == 0.0 && maxValue == 0.0)
                type = 0;
            else if (sparsity == 1.0 && minValue == maxValue)
                type = 1;
        }
        attr = new String[] { String.valueOf(type) };
    }
    if (opcode.equals(DataGen.SEQ_OPCODE)) {
        vs[0] = _unknownStats;
        vs[1] = _unknownStats;
        vs[2] = stats[Integer.parseInt(parts[2])];
    } else //general case
    {
        String inst2 = replaceInstructionPatch(inst);
        MRInstruction mrinst = MRInstructionParser.parseSingleInstruction(inst2);
        if (mrinst instanceof UnaryMRInstructionBase) {
            UnaryMRInstructionBase uinst = (UnaryMRInstructionBase) mrinst;
            vs[0] = uinst.input >= 0 ? stats[uinst.input] : _unknownStats;
            vs[1] = _unknownStats;
            vs[2] = stats[uinst.output];
            if (//scalar input, e.g., print
            vs[0] == null)
                vs[0] = _scalarStats;
            if (//scalar output
            vs[2] == null)
                vs[2] = _scalarStats;
            if (mrinst instanceof MMTSJMRInstruction) {
                String type = ((MMTSJMRInstruction) mrinst).getMMTSJType().toString();
                attr = new String[] { type };
            } else if (mrinst instanceof CM_N_COVInstruction) {
                if (opcode.equals("cm"))
                    attr = new String[] { parts[parts.length - 2] };
            } else if (mrinst instanceof GroupedAggregateInstruction) {
                if (opcode.equals("groupedagg")) {
                    AggregateOperationTypes type = CMOperator.getAggOpType(parts[2], parts[3]);
                    attr = new String[] { String.valueOf(type.ordinal()) };
                }
            }
        } else if (mrinst instanceof BinaryMRInstructionBase) {
            BinaryMRInstructionBase binst = (BinaryMRInstructionBase) mrinst;
            vs[0] = stats[binst.input1];
            vs[1] = stats[binst.input2];
            vs[2] = stats[binst.output];
            if (//scalar input, 
            vs[0] == null)
                vs[0] = _scalarStats;
            if (//scalar input, 
            vs[1] == null)
                vs[1] = _scalarStats;
            if (//scalar output
            vs[2] == null)
                vs[2] = _scalarStats;
            if (opcode.equals("rmempty")) {
                RemoveEmptyMRInstruction rbinst = (RemoveEmptyMRInstruction) mrinst;
                attr = new String[] { rbinst.isRemoveRows() ? "0" : "1" };
            }
        } else if (mrinst instanceof TernaryInstruction) {
            TernaryInstruction tinst = (TernaryInstruction) mrinst;
            vs[0] = stats[tinst.input1];
            vs[1] = stats[tinst.input2];
            vs[2] = stats[tinst.input3];
            if (//scalar input, 
            vs[0] == null)
                vs[0] = _scalarStats;
            if (//scalar input, 
            vs[1] == null)
                vs[1] = _scalarStats;
            if (//scalar input
            vs[2] == null)
                vs[2] = _scalarStats;
        } else if (mrinst instanceof PickByCountInstruction) {
            PickByCountInstruction pinst = (PickByCountInstruction) mrinst;
            vs[0] = stats[pinst.input1];
            vs[2] = stats[pinst.output];
            if (//scalar input, 
            vs[0] == null)
                vs[0] = _scalarStats;
            if (//scalar input, 
            vs[1] == null)
                vs[1] = _scalarStats;
            if (//scalar input
            vs[2] == null)
                vs[2] = _scalarStats;
        } else if (mrinst instanceof MapMultChainInstruction) {
            MapMultChainInstruction minst = (MapMultChainInstruction) mrinst;
            vs[0] = stats[minst.getInput1()];
            vs[1] = stats[minst.getInput2()];
            if (minst.getInput3() >= 0)
                vs[2] = stats[minst.getInput3()];
            if (//scalar input, 
            vs[0] == null)
                vs[0] = _scalarStats;
            if (//scalar input, 
            vs[1] == null)
                vs[1] = _scalarStats;
            if (//scalar input
            vs[2] == null)
                vs[2] = _scalarStats;
        }
    }
    //maintain var status (CP output always inmem)
    vs[2]._inmem = true;
    ret[0] = vs;
    ret[1] = attr;
    return ret;
}

Example 2 with MRInstruction

use of org.apache.sysml.runtime.instructions.mr.MRInstruction in project incubator-systemml by apache.

the class CostEstimator method maintainMRJobInstVariableStatistics.

private void maintainMRJobInstVariableStatistics(Instruction inst, HashMap<String, VarStats> stats) throws DMLRuntimeException {
    MRJobInstruction jobinst = (MRJobInstruction) inst;
    //input sizes (varname, index mapping)
    String[] inVars = jobinst.getInputVars();
    int index = -1;
    for (String varname : inVars) {
        VarStats vs = stats.get(varname);
        if (vs == null)
            vs = _unknownStats;
        stats.put(String.valueOf(++index), vs);
    }
    //rand output
    String rdInst = jobinst.getIv_randInstructions();
    if (rdInst != null && rdInst.length() > 0) {
        StringTokenizer st = new StringTokenizer(rdInst, Lop.INSTRUCTION_DELIMITOR);
        while (//foreach rand instruction
        st.hasMoreTokens()) {
            String[] parts = InstructionUtils.getInstructionParts(st.nextToken());
            byte outIndex = Byte.parseByte(parts[2]);
            long rlen = parts[3].contains(Lop.VARIABLE_NAME_PLACEHOLDER) ? -1 : UtilFunctions.parseToLong(parts[3]);
            long clen = parts[4].contains(Lop.VARIABLE_NAME_PLACEHOLDER) ? -1 : UtilFunctions.parseToLong(parts[4]);
            long brlen = Long.parseLong(parts[5]);
            long bclen = Long.parseLong(parts[6]);
            long nnz = (long) (Double.parseDouble(parts[9]) * rlen * clen);
            VarStats vs = new VarStats(rlen, clen, brlen, bclen, nnz, false);
            stats.put(String.valueOf(outIndex), vs);
        }
    }
    //compute intermediate result indices
    HashMap<Byte, MatrixCharacteristics> dims = new HashMap<Byte, MatrixCharacteristics>();
    //populate input indices
    for (Entry<String, VarStats> e : stats.entrySet()) {
        if (UtilFunctions.isIntegerNumber(e.getKey())) {
            byte ix = Byte.parseByte(e.getKey());
            VarStats vs = e.getValue();
            if (vs != null) {
                MatrixCharacteristics mc = new MatrixCharacteristics(vs._rlen, vs._clen, (int) vs._brlen, (int) vs._bclen, (long) vs._nnz);
                dims.put(ix, mc);
            }
        }
    }
    //compute dims for all instructions
    String[] instCat = new String[] { jobinst.getIv_randInstructions(), jobinst.getIv_recordReaderInstructions(), jobinst.getIv_instructionsInMapper(), jobinst.getIv_shuffleInstructions(), jobinst.getIv_aggInstructions(), jobinst.getIv_otherInstructions() };
    for (String linstCat : instCat) if (linstCat != null && linstCat.length() > 0) {
        String[] linst = linstCat.split(Instruction.INSTRUCTION_DELIM);
        for (String instStr : linst) {
            String instStr2 = replaceInstructionPatch(instStr);
            MRInstruction mrinst = MRInstructionParser.parseSingleInstruction(instStr2);
            MatrixCharacteristics.computeDimension(dims, mrinst);
        }
    }
    //create varstats if necessary
    for (Entry<Byte, MatrixCharacteristics> e : dims.entrySet()) {
        byte ix = e.getKey();
        if (!stats.containsKey(String.valueOf(ix))) {
            MatrixCharacteristics mc = e.getValue();
            VarStats vs = new VarStats(mc.getRows(), mc.getCols(), mc.getRowsPerBlock(), mc.getColsPerBlock(), mc.getNonZeros(), false);
            stats.put(String.valueOf(ix), vs);
        }
    }
    //map result indexes
    String[] outLabels = jobinst.getOutputVars();
    byte[] resultIndexes = jobinst.getIv_resultIndices();
    for (int i = 0; i < resultIndexes.length; i++) {
        String varname = outLabels[i];
        VarStats varvs = stats.get(String.valueOf(resultIndexes[i]));
        if (varvs == null) {
            varvs = stats.get(outLabels[i]);
        }
        varvs._inmem = false;
        stats.put(varname, varvs);
    }
}

Example 3 with MRInstruction

use of org.apache.sysml.runtime.instructions.mr.MRInstruction in project incubator-systemml by apache.

the class MapperBase method configure.

public void configure(JobConf job) {
    super.configure(job);
    //since one matrix file can occur multiple times in a statement
    try {
        representativeMatrixes = MRJobConfiguration.getInputMatrixIndexesInMapper(job);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    //get input converter information
    inputConverter = MRJobConfiguration.getInputConverter(job, representativeMatrixes.get(0));
    DataGenMRInstruction[] allDataGenIns;
    MRInstruction[] allMapperIns;
    ReblockInstruction[] allReblockIns;
    CSVReblockInstruction[] allCSVReblockIns;
    try {
        allDataGenIns = MRJobConfiguration.getDataGenInstructions(job);
        //parse the instructions on the matrices that this file represent
        allMapperIns = MRJobConfiguration.getInstructionsInMapper(job);
        //parse the reblock instructions on the matrices that this file represent
        allReblockIns = MRJobConfiguration.getReblockInstructions(job);
        allCSVReblockIns = MRJobConfiguration.getCSVReblockInstructions(job);
    } catch (DMLRuntimeException e) {
        throw new RuntimeException(e);
    }
    //get all the output indexes
    byte[] outputs = MRJobConfiguration.getOutputIndexesInMapper(job);
    //get the dimension of all the representative matrices
    rlens = new long[representativeMatrixes.size()];
    clens = new long[representativeMatrixes.size()];
    for (int i = 0; i < representativeMatrixes.size(); i++) {
        rlens[i] = MRJobConfiguration.getNumRows(job, representativeMatrixes.get(i));
        clens[i] = MRJobConfiguration.getNumColumns(job, representativeMatrixes.get(i));
    //	System.out.println("get dimension for "+representativeMatrixes.get(i)+": "+rlens[i]+", "+clens[i]);
    }
    //get the block sizes of the representative matrices
    brlens = new int[representativeMatrixes.size()];
    bclens = new int[representativeMatrixes.size()];
    for (int i = 0; i < representativeMatrixes.size(); i++) {
        brlens[i] = MRJobConfiguration.getNumRowsPerBlock(job, representativeMatrixes.get(i));
        bclens[i] = MRJobConfiguration.getNumColumnsPerBlock(job, representativeMatrixes.get(i));
    //	System.out.println("get blocksize for "+representativeMatrixes.get(i)+": "+brlens[i]+", "+bclens[i]);
    }
    rbounds = new long[representativeMatrixes.size()];
    cbounds = new long[representativeMatrixes.size()];
    lastblockrlens = new int[representativeMatrixes.size()];
    lastblockclens = new int[representativeMatrixes.size()];
    //calculate upper boundaries for key value pairs
    if (valueClass.equals(MatrixBlock.class)) {
        for (int i = 0; i < representativeMatrixes.size(); i++) {
            rbounds[i] = (long) Math.ceil((double) rlens[i] / (double) brlens[i]);
            cbounds[i] = (long) Math.ceil((double) clens[i] / (double) bclens[i]);
            lastblockrlens[i] = (int) (rlens[i] % brlens[i]);
            lastblockclens[i] = (int) (clens[i] % bclens[i]);
            if (lastblockrlens[i] == 0)
                lastblockrlens[i] = brlens[i];
            if (lastblockclens[i] == 0)
                lastblockclens[i] = bclens[i];
        /*
				 * what is this for????
				// DRB: the row indexes need to be fixed 
				rbounds[i] = rlens[i];*/
        }
    } else {
        for (int i = 0; i < representativeMatrixes.size(); i++) {
            rbounds[i] = rlens[i];
            cbounds[i] = clens[i];
            lastblockrlens[i] = 1;
            lastblockclens[i] = 1;
        //	System.out.println("get bound for "+representativeMatrixes.get(i)+": "+rbounds[i]+", "+cbounds[i]);
        }
    }
    //load data from distributed cache (if required, reuse if jvm_reuse)
    try {
        setupDistCacheFiles(job);
    } catch (IOException ex) {
        throw new RuntimeException(ex);
    }
    //collect unary instructions for each representative matrix
    HashSet<Byte> set = new HashSet<Byte>();
    for (int i = 0; i < representativeMatrixes.size(); i++) {
        set.clear();
        set.add(representativeMatrixes.get(i));
        //collect the relavent datagen instructions for this representative matrix
        ArrayList<DataGenMRInstruction> dataGensForThisMatrix = new ArrayList<DataGenMRInstruction>();
        if (allDataGenIns != null) {
            for (DataGenMRInstruction ins : allDataGenIns) {
                if (set.contains(ins.getInput())) {
                    dataGensForThisMatrix.add(ins);
                    set.add(ins.output);
                }
            }
        }
        if (dataGensForThisMatrix.size() > 1)
            throw new RuntimeException("only expects at most one rand instruction per input");
        if (dataGensForThisMatrix.isEmpty())
            dataGen_instructions.add(null);
        else
            dataGen_instructions.add(dataGensForThisMatrix.get(0));
        //collect the relavent instructions for this representative matrix
        ArrayList<MRInstruction> opsForThisMatrix = new ArrayList<MRInstruction>();
        if (allMapperIns != null) {
            for (MRInstruction ins : allMapperIns) {
                try {
                    /*
						boolean toAdd=true;
						for(byte input: ins.getInputIndexes())
							if(!set.contains(input))
							{
								toAdd=false;
								break;
							}
							*/
                    boolean toAdd = false;
                    for (byte input : ins.getInputIndexes()) if (set.contains(input)) {
                        toAdd = true;
                        break;
                    }
                    if (toAdd) {
                        opsForThisMatrix.add(ins);
                        set.add(ins.output);
                    }
                } catch (DMLRuntimeException e) {
                    throw new RuntimeException(e);
                }
            }
        }
        mapper_instructions.add(opsForThisMatrix);
        //collect the relavent reblock instructions for this representative matrix
        ArrayList<ReblockInstruction> reblocksForThisMatrix = new ArrayList<ReblockInstruction>();
        if (allReblockIns != null) {
            for (ReblockInstruction ins : allReblockIns) {
                if (set.contains(ins.input)) {
                    reblocksForThisMatrix.add(ins);
                    set.add(ins.output);
                }
            }
        }
        reblock_instructions.add(reblocksForThisMatrix);
        //collect the relavent reblock instructions for this representative matrix
        ArrayList<CSVReblockInstruction> csvReblocksForThisMatrix = new ArrayList<CSVReblockInstruction>();
        if (allCSVReblockIns != null) {
            for (CSVReblockInstruction ins : allCSVReblockIns) {
                if (set.contains(ins.input)) {
                    csvReblocksForThisMatrix.add(ins);
                    set.add(ins.output);
                }
            }
        }
        csv_reblock_instructions.add(csvReblocksForThisMatrix);
        //collect the output indexes for this representative matrix
        ArrayList<Byte> outsForThisMatrix = new ArrayList<Byte>();
        for (byte output : outputs) {
            if (set.contains(output))
                outsForThisMatrix.add(output);
        }
        outputIndexes.add(outsForThisMatrix);
    }
}

Example 4 with MRInstruction

use of org.apache.sysml.runtime.instructions.mr.MRInstruction in project incubator-systemml by apache.

the class MapperBase method allowsFilterEmptyInputBlocks.

/**
	 * Determines if empty blocks can be discarded on map input. Conceptually, this is true
	 * if the individual instruction don't need to output empty blocks and if they are sparsesafe.
	 * 
	 * @return true if empty blocks can be discarded on map input
	 */
public boolean allowsFilterEmptyInputBlocks() {
    boolean ret = true;
    int count = 0;
    if (ret && mapper_instructions != null)
        for (ArrayList<MRInstruction> vinst : mapper_instructions) for (MRInstruction inst : vinst) {
            ret &= (inst instanceof AggregateBinaryInstruction && !((AggregateBinaryInstruction) inst).getOutputEmptyBlocks()) || (inst instanceof PMMJMRInstruction && !((PMMJMRInstruction) inst).getOutputEmptyBlocks());
            //ensure that mapper instructions exists
            count++;
        }
    return ret && count > 0;
}

Example 5 with MRInstruction

use of org.apache.sysml.runtime.instructions.mr.MRInstruction in project incubator-systemml by apache.

the class ProgramConverter method cloneInstruction.

public static Instruction cloneInstruction(Instruction oInst, long pid, boolean plain, boolean cpFunctions) throws DMLRuntimeException {
    Instruction inst = null;
    String tmpString = oInst.toString();
    try {
        if (oInst instanceof CPInstruction || oInst instanceof SPInstruction || oInst instanceof MRInstruction || oInst instanceof GPUInstruction) {
            if (oInst instanceof FunctionCallCPInstruction && cpFunctions) {
                FunctionCallCPInstruction tmp = (FunctionCallCPInstruction) oInst;
                if (!plain) {
                    //safe replacement because target variables might include the function name
                    //note: this is no update-in-place in order to keep the original function name as basis
                    tmpString = tmp.updateInstStringFunctionName(tmp.getFunctionName(), tmp.getFunctionName() + CP_CHILD_THREAD + pid);
                }
            //otherwise: preserve function name
            }
            inst = InstructionParser.parseSingleInstruction(tmpString);
        } else if (oInst instanceof MRJobInstruction) {
            //clone via copy constructor
            inst = new MRJobInstruction((MRJobInstruction) oInst);
        } else
            throw new DMLRuntimeException("Failed to clone instruction: " + oInst);
    } catch (Exception ex) {
        throw new DMLRuntimeException(ex);
    }
    //save replacement of thread id references in instructions
    inst = saveReplaceThreadID(inst, ProgramConverter.CP_ROOT_THREAD_ID, ProgramConverter.CP_CHILD_THREAD + pid);
    return inst;
}