Examples with Instruction org.apache.sysml.runtime.instructions.Instruction used on opensource projects

Example 96 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project incubator-systemml by apache.

the class OptimizerRuleBased method recompileLIX.

protected void recompileLIX(OptNode n, LocalVariableMap vars) {
    Hop h = OptTreeConverter.getAbstractPlanMapping().getMappedHop(n.getID());
    // set forced exec type
    h.setForcedExecType(LopProperties.ExecType.CP);
    n.setExecType(ExecType.CP);
    // recompile parent pb
    long pid = OptTreeConverter.getAbstractPlanMapping().getMappedParentID(n.getID());
    OptNode nParent = OptTreeConverter.getAbstractPlanMapping().getOptNode(pid);
    Object[] o = OptTreeConverter.getAbstractPlanMapping().getMappedProg(pid);
    StatementBlock sb = (StatementBlock) o[0];
    ProgramBlock pb = (ProgramBlock) o[1];
    // keep modified estimated of partitioned rix (in same dag as lix)
    HashMap<Hop, Double> estRix = getPartitionedRIXEstimates(nParent);
    // construct new instructions
    ArrayList<Instruction> newInst = Recompiler.recompileHopsDag(sb, sb.getHops(), vars, null, false, false, 0);
    pb.setInstructions(newInst);
    // reset all rix estimated (modified by recompile)
    resetPartitionRIXEstimates(estRix);
    // set new mem estimate (last, otherwise overwritten from recompile)
    h.setMemEstimate(_rm - 1);
}

Example 97 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project incubator-systemml by apache.

the class ProgramRecompiler method createNestedParallelismToInstructionSet.

// /////
// additional general-purpose functionalities
protected static ArrayList<Instruction> createNestedParallelismToInstructionSet(String iterVar, String offset) {
    // create instruction string
    StringBuilder sb = new StringBuilder("CP" + Lop.OPERAND_DELIMITOR + "+" + Lop.OPERAND_DELIMITOR);
    sb.append(iterVar);
    sb.append(Lop.DATATYPE_PREFIX + "SCALAR" + Lop.VALUETYPE_PREFIX + "INT" + Lop.OPERAND_DELIMITOR);
    sb.append(offset);
    sb.append(Lop.DATATYPE_PREFIX + "SCALAR" + Lop.VALUETYPE_PREFIX + "INT" + Lop.OPERAND_DELIMITOR);
    sb.append(iterVar);
    sb.append(Lop.DATATYPE_PREFIX + "SCALAR" + Lop.VALUETYPE_PREFIX + "INT");
    String str = sb.toString();
    // create instruction set
    ArrayList<Instruction> tmp = new ArrayList<>();
    Instruction inst = BinaryCPInstruction.parseInstruction(str);
    tmp.add(inst);
    return tmp;
}

Example 98 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project incubator-systemml by apache.

the class ExternalFunctionProgramBlock method execute.

/**
 * Method to be invoked to execute instructions for the external function
 * invocation
 */
@Override
public void execute(ExecutionContext ec) {
    _runID = _idSeq.getNextID();
    changeTmpInput(_runID, ec);
    changeTmpOutput(_runID);
    // export input variables to HDFS (see RunMRJobs)
    ArrayList<DataIdentifier> inputParams = null;
    try {
        inputParams = getInputParams();
        for (DataIdentifier di : inputParams) {
            Data d = ec.getVariable(di.getName());
            if (d.getDataType().isMatrix())
                ((MatrixObject) d).exportData();
        }
    } catch (Exception e) {
        throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error exporting input variables to HDFS", e);
    }
    // convert block to cell
    if (block2CellInst != null) {
        ArrayList<Instruction> tempInst = new ArrayList<>();
        tempInst.addAll(block2CellInst);
        try {
            this.executeInstructions(tempInst, ec);
        } catch (Exception e) {
            throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error executing " + tempInst.toString(), e);
        }
    }
    // now execute package function
    for (int i = 0; i < _inst.size(); i++) {
        try {
            if (_inst.get(i) instanceof ExternalFunctionInvocationInstruction)
                ((ExternalFunctionInvocationInstruction) _inst.get(i)).processInstruction(ec);
        } catch (Exception e) {
            throw new DMLRuntimeException(this.printBlockErrorLocation() + "Failed to execute instruction " + _inst.get(i).toString(), e);
        }
    }
    // convert cell to block
    if (cell2BlockInst != null) {
        ArrayList<Instruction> tempInst = new ArrayList<>();
        try {
            tempInst.clear();
            tempInst.addAll(cell2BlockInst);
            this.executeInstructions(tempInst, ec);
        } catch (Exception e) {
            throw new DMLRuntimeException(this.printBlockErrorLocation() + "Failed to execute instruction " + cell2BlockInst.toString(), e);
        }
    }
    // check return values
    checkOutputParameters(ec.getVariables());
}

Example 99 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project incubator-systemml by apache.

the class ExternalFunctionProgramBlock method createFunctionObject.

@SuppressWarnings("unchecked")
protected PackageFunction createFunctionObject(String className, String configFile) {
    try {
        // create instance of package function
        Class<Instruction> cla = (Class<Instruction>) Class.forName(className);
        Object o = cla.newInstance();
        if (!(o instanceof PackageFunction))
            throw new DMLRuntimeException(this.printBlockErrorLocation() + "Class is not of type PackageFunction");
        PackageFunction fun = (PackageFunction) o;
        // configure package function
        fun.setConfiguration(configFile);
        fun.setBaseDir(_baseDir);
        return fun;
    } catch (Exception e) {
        throw new DMLRuntimeException(this.printBlockErrorLocation() + "Error instantiating package function ", e);
    }
}

Example 100 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project incubator-systemml by apache.

the class ExternalFunctionProgramBlock method getCell2BlockInstructions.

/**
 * Method to generate a reblock job to convert the cell representation into block representation
 *
 * @param outputParams list out output data identifiers
 * @param blockedFileNames map of blocked file names
 * @return list of instructions
 */
private ArrayList<Instruction> getCell2BlockInstructions(ArrayList<DataIdentifier> outputParams, HashMap<String, String> blockedFileNames) {
    ArrayList<Instruction> c2binst = null;
    // list of matrices that need to be reblocked
    ArrayList<DataIdentifier> matrices = new ArrayList<>();
    ArrayList<DataIdentifier> matricesNoReblock = new ArrayList<>();
    // identify outputs that are matrices
    for (int i = 0; i < outputParams.size(); i++) {
        if (outputParams.get(i).getDataType().isMatrix()) {
            if (_skipOutReblock.contains(outputParams.get(i).getName()))
                matricesNoReblock.add(outputParams.get(i));
            else
                matrices.add(outputParams.get(i));
        }
    }
    if (!matrices.isEmpty()) {
        c2binst = new ArrayList<>();
        MRJobInstruction reblkInst = new MRJobInstruction(JobType.REBLOCK);
        TreeMap<Integer, ArrayList<String>> MRJobLineNumbers = null;
        if (DMLScript.ENABLE_DEBUG_MODE) {
            MRJobLineNumbers = new TreeMap<>();
        }
        ArrayList<String> inLabels = new ArrayList<>();
        ArrayList<String> outLabels = new ArrayList<>();
        String[] outputs = new String[matrices.size()];
        byte[] resultIndex = new byte[matrices.size()];
        String reblock = "";
        // Keep a copy of a single MR reblock instruction
        String reblockStr = "";
        String scratchSpaceLoc = ConfigurationManager.getScratchSpace();
        try {
            // create a RBLK job that transforms each output matrix from cell to block
            for (int i = 0; i < matrices.size(); i++) {
                inLabels.add(matrices.get(i).getName());
                outLabels.add(matrices.get(i).getName() + "_extFnOutput");
                outputs[i] = scratchSpaceLoc + Lop.FILE_SEPARATOR + Lop.PROCESS_PREFIX + DMLScript.getUUID() + Lop.FILE_SEPARATOR + _otherParams.get(ExternalFunctionStatement.CLASS_NAME) + _runID + "_" + i + "Output";
                blockedFileNames.put(matrices.get(i).getName(), outputs[i]);
                // (matrices.size()+i);
                resultIndex[i] = (byte) i;
                if (i > 0)
                    reblock += Lop.INSTRUCTION_DELIMITOR;
                reblock += "MR" + Lop.OPERAND_DELIMITOR + "rblk" + Lop.OPERAND_DELIMITOR + i + Lop.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lop.VALUETYPE_PREFIX + matrices.get(i).getValueType() + Lop.OPERAND_DELIMITOR + i + Lop.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lop.VALUETYPE_PREFIX + matrices.get(i).getValueType() + Lop.OPERAND_DELIMITOR + ConfigurationManager.getBlocksize() + Lop.OPERAND_DELIMITOR + ConfigurationManager.getBlocksize() + Lop.OPERAND_DELIMITOR + "true";
                if (DMLScript.ENABLE_DEBUG_MODE) {
                    // Create a copy of reblock instruction but as a single instruction (FOR DEBUGGER)
                    reblockStr = "MR" + Lop.OPERAND_DELIMITOR + "rblk" + Lop.OPERAND_DELIMITOR + i + Lop.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lop.VALUETYPE_PREFIX + matrices.get(i).getValueType() + Lop.OPERAND_DELIMITOR + i + Lop.DATATYPE_PREFIX + matrices.get(i).getDataType() + Lop.VALUETYPE_PREFIX + matrices.get(i).getValueType() + Lop.OPERAND_DELIMITOR + ConfigurationManager.getBlocksize() + Lop.OPERAND_DELIMITOR + ConfigurationManager.getBlocksize() + Lop.OPERAND_DELIMITOR + "true";
                    // Set MR reblock instruction line number (FOR DEBUGGER)
                    if (!MRJobLineNumbers.containsKey(matrices.get(i).getBeginLine())) {
                        MRJobLineNumbers.put(matrices.get(i).getBeginLine(), new ArrayList<String>());
                    }
                    MRJobLineNumbers.get(matrices.get(i).getBeginLine()).add(reblockStr);
                }
                // create metadata instructions to populate symbol table
                // with variables that hold blocked matrices
                Instruction createInst = VariableCPInstruction.prepareCreateMatrixVariableInstruction(outLabels.get(i), outputs[i], false, OutputInfo.outputInfoToString(OutputInfo.BinaryBlockOutputInfo));
                createInst.setLocation(matrices.get(i));
                c2binst.add(createInst);
            }
            reblkInst.setReBlockInstructions(inLabels.toArray(new String[inLabels.size()]), "", reblock, "", outLabels.toArray(new String[inLabels.size()]), resultIndex, 1, 1);
            c2binst.add(reblkInst);
            // generate instructions that rename the output variables of REBLOCK job
            Instruction cpInst = null, rmInst = null;
            for (int i = 0; i < matrices.size(); i++) {
                cpInst = VariableCPInstruction.prepareCopyInstruction(outLabels.get(i), matrices.get(i).getName());
                rmInst = VariableCPInstruction.prepareRemoveInstruction(outLabels.get(i));
                cpInst.setLocation(matrices.get(i));
                rmInst.setLocation(matrices.get(i));
                c2binst.add(cpInst);
                c2binst.add(rmInst);
            // c2binst.add(CPInstructionParser.parseSingleInstruction("CP" + Lops.OPERAND_DELIMITOR + "cpvar"+Lops.OPERAND_DELIMITOR+ outLabels.get(i) + Lops.OPERAND_DELIMITOR + matrices.get(i).getName()));
            }
        } catch (Exception e) {
            throw new RuntimeException(this.printBlockErrorLocation() + "error generating instructions", e);
        }
        // LOGGING instructions
        if (LOG.isTraceEnabled()) {
            LOG.trace("\n--- Cell-2-Block Instructions ---");
            for (Instruction i : c2binst) {
                LOG.trace(i.toString());
            }
            LOG.trace("----------------------------------");
        }
    }
    // null if no output matrices
    return c2binst;
}