Examples with Instruction org.apache.sysml.runtime.instructions.Instruction used on opensource projects

Example 61 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project incubator-systemml by apache.

the class ExternalFunctionProgramBlock method getCell2BlockInstructions.

/**
 * Method to generate a reblock job to convert the cell representation into block representation
 *
 * @param outputParams list out output data identifiers
 * @param blockedFileNames map of blocked file names
 * @return list of instructions
 */
private ArrayList<Instruction> getCell2BlockInstructions(ArrayList<DataIdentifier> outputParams, HashMap<String, String> blockedFileNames) {
    ArrayList<Instruction> c2binst = null;
    // list of matrices that need to be reblocked
    ArrayList<DataIdentifier> matrices = new ArrayList<>();
    ArrayList<DataIdentifier> matricesNoReblock = new ArrayList<>();
    // identify outputs that are matrices
    for (int i = 0; i < outputParams.size(); i++) {
        if (outputParams.get(i).getDataType().isMatrix()) {
            if (_skipOutReblock.contains(outputParams.get(i).getName()))
                matricesNoReblock.add(outputParams.get(i));
            else
                matrices.add(outputParams.get(i));
        }
    }
    if (!matrices.isEmpty()) {
        c2binst = new ArrayList<>();
        MRJobInstruction reblkInst = new MRJobInstruction(JobType.REBLOCK);
        TreeMap<Integer, ArrayList<String>> MRJobLineNumbers = null;
        if (DMLScript.ENABLE_DEBUG_MODE) {
            MRJobLineNumbers = new TreeMap<>();
        }
        ArrayList<String> inLabels = new ArrayList<>();
        ArrayList<String> outLabels = new ArrayList<>();
        String[] outputs = new String[matrices.size()];
        byte[] resultIndex = new byte[matrices.size()];
        String reblock = "";
        // Keep a copy of a single MR reblock instruction
        String reblockStr = "";
        String scratchSpaceLoc = ConfigurationManager.getScratchSpace();
        try {
            // create a RBLK job that transforms each output matrix from cell to block
            for (int i = 0; i < matrices.size(); i++) {
                inLabels.add(matrices.get(i).getName());
                outLabels.add(matrices.get(i).getName() + "_extFnOutput");
                outputs[i] = scratchSpaceLoc + Lop.FILE_SEPARATOR + Lop.PROCESS_PREFIX + DMLScript.getUUID() + Lop.FILE_SEPARATOR + _otherParams.get(ExternalFunctionStatement.CLASS_NAME) + _runID + "_" + i + "Output";
                blockedFileNames.put(matrices.get(i).getName(), outputs[i]);
                // (matrices.size()+i);
                resultIndex[i] = (byte) i;
                if (i > 0)
                    reblock += Lop.INSTRUCTION_DELIMITOR;
                reblock += "MR" + Lop.OPERAND_DELIMITOR + "rblk" + Lop.OPERAND_DELIMITOR + i + Lop.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lop.VALUETYPE_PREFIX + matrices.get(i).getValueType() + Lop.OPERAND_DELIMITOR + i + Lop.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lop.VALUETYPE_PREFIX + matrices.get(i).getValueType() + Lop.OPERAND_DELIMITOR + ConfigurationManager.getBlocksize() + Lop.OPERAND_DELIMITOR + ConfigurationManager.getBlocksize() + Lop.OPERAND_DELIMITOR + "true";
                if (DMLScript.ENABLE_DEBUG_MODE) {
                    // Create a copy of reblock instruction but as a single instruction (FOR DEBUGGER)
                    reblockStr = "MR" + Lop.OPERAND_DELIMITOR + "rblk" + Lop.OPERAND_DELIMITOR + i + Lop.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lop.VALUETYPE_PREFIX + matrices.get(i).getValueType() + Lop.OPERAND_DELIMITOR + i + Lop.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lop.VALUETYPE_PREFIX + matrices.get(i).getValueType() + Lop.OPERAND_DELIMITOR + ConfigurationManager.getBlocksize() + Lop.OPERAND_DELIMITOR + ConfigurationManager.getBlocksize() + Lop.OPERAND_DELIMITOR + "true";
                    // Set MR reblock instruction line number (FOR DEBUGGER)
                    if (!MRJobLineNumbers.containsKey(matrices.get(i).getBeginLine())) {
                        MRJobLineNumbers.put(matrices.get(i).getBeginLine(), new ArrayList<String>());
                    }
                    MRJobLineNumbers.get(matrices.get(i).getBeginLine()).add(reblockStr);
                }
                // create metadata instructions to populate symbol table
                // with variables that hold blocked matrices
                Instruction createInst = VariableCPInstruction.prepareCreateMatrixVariableInstruction(outLabels.get(i), outputs[i], false, OutputInfo.outputInfoToString(OutputInfo.BinaryBlockOutputInfo));
                createInst.setLocation(matrices.get(i));
                c2binst.add(createInst);
            }
            reblkInst.setReBlockInstructions(inLabels.toArray(new String[inLabels.size()]), "", reblock, "", outLabels.toArray(new String[inLabels.size()]), resultIndex, 1, 1);
            c2binst.add(reblkInst);
            // generate instructions that rename the output variables of REBLOCK job
            Instruction cpInst = null, rmInst = null;
            for (int i = 0; i < matrices.size(); i++) {
                cpInst = VariableCPInstruction.prepareCopyInstruction(outLabels.get(i), matrices.get(i).getName());
                rmInst = VariableCPInstruction.prepareRemoveInstruction(outLabels.get(i));
                cpInst.setLocation(matrices.get(i));
                rmInst.setLocation(matrices.get(i));
                c2binst.add(cpInst);
                c2binst.add(rmInst);
            // c2binst.add(CPInstructionParser.parseSingleInstruction("CP" + Lops.OPERAND_DELIMITOR + "cpvar"+Lops.OPERAND_DELIMITOR+ outLabels.get(i) + Lops.OPERAND_DELIMITOR + matrices.get(i).getName()));
            }
        } catch (Exception e) {
            throw new RuntimeException(this.printBlockErrorLocation() + "error generating instructions", e);
        }
        // LOGGING instructions
        if (LOG.isTraceEnabled()) {
            LOG.trace("\n--- Cell-2-Block Instructions ---");
            for (Instruction i : c2binst) {
                LOG.trace(i.toString());
            }
            LOG.trace("----------------------------------");
        }
    }
    // null if no output matrices
    return c2binst;
}

Example 62 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project incubator-systemml by apache.

the class ProgramBlock method executeSingleInstruction.

private void executeSingleInstruction(Instruction currInst, ExecutionContext ec) {
    try {
        // start time measurement for statistics
        long t0 = (DMLScript.STATISTICS || LOG.isTraceEnabled()) ? System.nanoTime() : 0;
        // pre-process instruction (debug state, inst patching, listeners)
        Instruction tmp = currInst.preprocessInstruction(ec);
        // process actual instruction
        tmp.processInstruction(ec);
        // post-process instruction (debug)
        tmp.postprocessInstruction(ec);
        // maintain aggregate statistics
        if (DMLScript.STATISTICS) {
            Statistics.maintainCPHeavyHitters(tmp.getExtendedOpcode(), System.nanoTime() - t0);
        }
        // optional trace information (instruction and runtime)
        if (LOG.isTraceEnabled()) {
            long t1 = System.nanoTime();
            String time = String.format("%.3f", ((double) t1 - t0) / 1000000000);
            LOG.trace("Instruction: " + tmp + " (executed in " + time + "s).");
        }
        // variables in symbol table (for tracking source of wrong representation)
        if (CHECK_MATRIX_SPARSITY) {
            checkSparsity(tmp, ec.getVariables());
        }
    } catch (Exception e) {
        if (!DMLScript.ENABLE_DEBUG_MODE) {
            if (e instanceof DMLScriptException)
                throw (DMLScriptException) e;
            else
                throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error evaluating instruction: " + currInst.toString(), e);
        } else {
            ec.handleDebugException(e);
        }
    }
}

Example 63 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project incubator-systemml by apache.

the class ProgramBlock method executePredicate.

/**
 * Executes given predicate instructions (incl recompilation if required)
 *
 * @param inst list of instructions
 * @param hops high-level operator
 * @param requiresRecompile true if requires recompile
 * @param retType value type of the return type
 * @param ec execution context
 * @return scalar object
 */
public ScalarObject executePredicate(ArrayList<Instruction> inst, Hop hops, boolean requiresRecompile, ValueType retType, ExecutionContext ec) {
    ArrayList<Instruction> tmp = inst;
    // dynamically recompile instructions if enabled and required
    try {
        long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0;
        if (ConfigurationManager.isDynamicRecompilation() && requiresRecompile) {
            tmp = Recompiler.recompileHopsDag(hops, ec.getVariables(), null, false, true, _tid);
            tmp = JMLCUtils.cleanupRuntimeInstructions(tmp, PRED_VAR);
        }
        if (DMLScript.STATISTICS) {
            long t1 = System.nanoTime();
            Statistics.incrementHOPRecompileTime(t1 - t0);
            if (tmp != inst)
                Statistics.incrementHOPRecompilePred();
        }
    } catch (Exception ex) {
        throw new DMLRuntimeException("Unable to recompile predicate instructions.", ex);
    }
    // actual instruction execution
    return executePredicateInstructions(tmp, retType, ec);
}

Example 64 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project incubator-systemml by apache.

the class SPInstruction method preprocessInstruction.

@Override
public Instruction preprocessInstruction(ExecutionContext ec) {
    // default pre-process behavior (e.g., debug state)
    Instruction tmp = super.preprocessInstruction(ec);
    // instruction patching
    if (// update labels only if required
    tmp.requiresLabelUpdate()) {
        // note: no exchange of updated instruction as labels might change in the general case
        String updInst = RunMRJobs.updateLabels(tmp.toString(), ec.getVariables());
        tmp = SPInstructionParser.parseSingleInstruction(updInst);
    }
    return tmp;
}

Example 65 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project incubator-systemml by apache.

the class MRJobConfiguration method computeMatrixCharacteristics.

/**
 * NOTE: this method needs to be in-sync with MRBaseForCommonInstructions.processOneInstruction,
 * otherwise, the latter will potentially fail with missing dimension information.
 *
 * @param job job configuration
 * @param inputIndexes array of byte indexes
 * @param dataGenInstructions data gen instructions as a string
 * @param instructionsInMapper instruction in mapper as a string
 * @param reblockInstructions reblock instructions as a string
 * @param aggInstructionsInReducer aggregate instructions in reducer as a string
 * @param aggBinInstructions binary aggregate instructions as a string
 * @param otherInstructionsInReducer other instructions in reducer as a string
 * @param resultIndexes array of byte result indexes
 * @param mapOutputIndexes set of map output indexes
 * @param forMMCJ ?
 * @return reducer groups
 */
public static MatrixChar_N_ReducerGroups computeMatrixCharacteristics(JobConf job, byte[] inputIndexes, String dataGenInstructions, String instructionsInMapper, String reblockInstructions, String aggInstructionsInReducer, String aggBinInstructions, String otherInstructionsInReducer, byte[] resultIndexes, HashSet<Byte> mapOutputIndexes, boolean forMMCJ) {
    HashSet<Byte> intermediateMatrixIndexes = new HashSet<>();
    HashMap<Byte, MatrixCharacteristics> dims = new HashMap<>();
    for (byte i : inputIndexes) {
        MatrixCharacteristics dim = new MatrixCharacteristics(getNumRows(job, i), getNumColumns(job, i), getNumRowsPerBlock(job, i), getNumColumnsPerBlock(job, i), getNumNonZero(job, i));
        dims.put(i, dim);
    }
    DataGenMRInstruction[] dataGenIns = null;
    dataGenIns = MRInstructionParser.parseDataGenInstructions(dataGenInstructions);
    if (dataGenIns != null) {
        for (DataGenMRInstruction ins : dataGenIns) {
            MatrixCharacteristics.computeDimension(dims, ins);
        }
    }
    MRInstruction[] insMapper = MRInstructionParser.parseMixedInstructions(instructionsInMapper);
    if (insMapper != null) {
        for (MRInstruction ins : insMapper) {
            MatrixCharacteristics.computeDimension(dims, ins);
            if (ins instanceof UnaryMRInstructionBase) {
                UnaryMRInstructionBase tempIns = (UnaryMRInstructionBase) ins;
                setIntermediateMatrixCharactristics(job, tempIns.input, dims.get(tempIns.input));
                intermediateMatrixIndexes.add(tempIns.input);
            } else if (ins instanceof AppendMInstruction) {
                AppendMInstruction tempIns = (AppendMInstruction) ins;
                setIntermediateMatrixCharactristics(job, tempIns.input1, dims.get(tempIns.input1));
                intermediateMatrixIndexes.add(tempIns.input1);
            } else if (ins instanceof AppendGInstruction) {
                AppendGInstruction tempIns = (AppendGInstruction) ins;
                setIntermediateMatrixCharactristics(job, tempIns.input1, dims.get(tempIns.input1));
                intermediateMatrixIndexes.add(tempIns.input1);
            } else if (ins instanceof BinaryMInstruction) {
                BinaryMInstruction tempIns = (BinaryMInstruction) ins;
                setIntermediateMatrixCharactristics(job, tempIns.input1, dims.get(tempIns.input1));
                intermediateMatrixIndexes.add(tempIns.input1);
            } else if (ins instanceof AggregateBinaryInstruction) {
                AggregateBinaryInstruction tempIns = (AggregateBinaryInstruction) ins;
                setIntermediateMatrixCharactristics(job, tempIns.input1, dims.get(tempIns.input1));
                // TODO
                intermediateMatrixIndexes.add(tempIns.input1);
            } else if (ins instanceof MapMultChainInstruction) {
                MapMultChainInstruction tempIns = (MapMultChainInstruction) ins;
                setIntermediateMatrixCharactristics(job, tempIns.getInput1(), dims.get(tempIns.getInput2()));
                intermediateMatrixIndexes.add(tempIns.getInput1());
            } else if (ins instanceof PMMJMRInstruction) {
                PMMJMRInstruction tempIns = (PMMJMRInstruction) ins;
                setIntermediateMatrixCharactristics(job, tempIns.input2, dims.get(tempIns.input2));
                intermediateMatrixIndexes.add(tempIns.input2);
            }
        }
    }
    ReblockInstruction[] reblockIns = MRInstructionParser.parseReblockInstructions(reblockInstructions);
    if (reblockIns != null) {
        for (ReblockInstruction ins : reblockIns) {
            MatrixCharacteristics.computeDimension(dims, ins);
            setMatrixCharactristicsForReblock(job, ins.output, dims.get(ins.output));
        }
    }
    Instruction[] aggIns = MRInstructionParser.parseAggregateInstructions(aggInstructionsInReducer);
    if (aggIns != null) {
        for (Instruction ins : aggIns) {
            MatrixCharacteristics.computeDimension(dims, (MRInstruction) ins);
            // if instruction's output is not in resultIndexes, then add its dimensions to jobconf
            MRInstruction mrins = (MRInstruction) ins;
            boolean found = false;
            for (byte b : resultIndexes) {
                if (b == mrins.output) {
                    found = true;
                    break;
                }
            }
            if (!found) {
                setIntermediateMatrixCharactristics(job, mrins.output, dims.get(mrins.output));
                intermediateMatrixIndexes.add(mrins.output);
            }
        }
    }
    long numReduceGroups = 0;
    AggregateBinaryInstruction[] aggBinIns = getAggregateBinaryInstructions(job);
    if (aggBinIns != null) {
        for (AggregateBinaryInstruction ins : aggBinIns) {
            MatrixCharacteristics dim1 = dims.get(ins.input1);
            MatrixCharacteristics dim2 = dims.get(ins.input2);
            setMatrixCharactristicsForBinAgg(job, ins.input1, dim1);
            setMatrixCharactristicsForBinAgg(job, ins.input2, dim2);
            MatrixCharacteristics.computeDimension(dims, ins);
            if (// there will be only one aggbin operation for MMCJ
            forMMCJ)
                numReduceGroups = (long) Math.ceil((double) dim1.getCols() / (double) dim1.getColsPerBlock());
        }
    }
    if (!forMMCJ) {
        // store the skylines
        ArrayList<Long> xs = new ArrayList<>(mapOutputIndexes.size());
        ArrayList<Long> ys = new ArrayList<>(mapOutputIndexes.size());
        for (byte idx : mapOutputIndexes) {
            MatrixCharacteristics dim = dims.get(idx);
            long x = (long) Math.ceil((double) dim.getRows() / (double) dim.getRowsPerBlock());
            long y = (long) Math.ceil((double) dim.getCols() / (double) dim.getColsPerBlock());
            int i = 0;
            boolean toadd = true;
            while (i < xs.size()) {
                if ((x >= xs.get(i) && y > ys.get(i)) || (x > xs.get(i) && y >= ys.get(i))) {
                    // remove any included x's and y's
                    xs.remove(i);
                    ys.remove(i);
                } else if (// if included in others, stop
                x <= xs.get(i) && y <= ys.get(i)) {
                    toadd = false;
                    break;
                } else
                    i++;
            }
            if (toadd) {
                xs.add(x);
                ys.add(y);
            }
        }
        // sort by x
        TreeMap<Long, Long> map = new TreeMap<>();
        for (int i = 0; i < xs.size(); i++) map.put(xs.get(i), ys.get(i));
        numReduceGroups = 0;
        // compute area
        long prev = 0;
        for (Entry<Long, Long> e : map.entrySet()) {
            numReduceGroups += (e.getKey() - prev) * e.getValue();
            prev = e.getKey();
        }
    }
    MRInstruction[] insReducer = MRInstructionParser.parseMixedInstructions(otherInstructionsInReducer);
    if (insReducer != null) {
        for (MRInstruction ins : insReducer) {
            MatrixCharacteristics.computeDimension(dims, ins);
            if (ins instanceof UnaryMRInstructionBase) {
                UnaryMRInstructionBase tempIns = (UnaryMRInstructionBase) ins;
                setIntermediateMatrixCharactristics(job, tempIns.input, dims.get(tempIns.input));
                intermediateMatrixIndexes.add(tempIns.input);
            } else if (ins instanceof RemoveEmptyMRInstruction) {
                RemoveEmptyMRInstruction tempIns = (RemoveEmptyMRInstruction) ins;
                setIntermediateMatrixCharactristics(job, tempIns.input1, dims.get(tempIns.input1));
                intermediateMatrixIndexes.add(tempIns.input1);
            }
            // if instruction's output is not in resultIndexes, then add its dimensions to jobconf
            boolean found = false;
            for (byte b : resultIndexes) {
                if (b == ins.output) {
                    found = true;
                    break;
                }
            }
            if (!found) {
                setIntermediateMatrixCharactristics(job, ins.output, dims.get(ins.output));
                intermediateMatrixIndexes.add(ins.output);
            }
        }
    }
    setIntermediateMatrixIndexes(job, intermediateMatrixIndexes);
    for (byte tag : mapOutputIndexes) setMatrixCharactristicsForMapperOutput(job, tag, dims.get(tag));
    MatrixCharacteristics[] stats = new MatrixCharacteristics[resultIndexes.length];
    MatrixCharacteristics resultDims;
    for (int i = 0; i < resultIndexes.length; i++) {
        resultDims = dims.get(resultIndexes[i]);
        stats[i] = resultDims;
        setMatrixCharactristicsForOutput(job, resultIndexes[i], stats[i]);
    }
    return new MatrixChar_N_ReducerGroups(stats, numReduceGroups);
}