Examples with CPInstruction org.apache.sysml.runtime.instructions.cp.CPInstruction used on opensource projects

Example 21 with CPInstruction

use of org.apache.sysml.runtime.instructions.cp.CPInstruction in project incubator-systemml by apache.

the class CPInstructionParser method parseSingleInstruction.

public static CPInstruction parseSingleInstruction(String str) {
    if (str == null || str.isEmpty())
        return null;
    CPType cptype = InstructionUtils.getCPType(str);
    if (cptype == null)
        throw new DMLRuntimeException("Unable derive cptype for instruction: " + str);
    CPInstruction cpinst = parseSingleInstruction(cptype, str);
    if (cpinst == null)
        throw new DMLRuntimeException("Unable to parse instruction: " + str);
    return cpinst;
}

Example 22 with CPInstruction

use of org.apache.sysml.runtime.instructions.cp.CPInstruction in project systemml by apache.

the class ProgramConverter method serializeInstructions.

@SuppressWarnings("all")
private static String serializeInstructions(ArrayList<Instruction> inst, HashMap<String, byte[]> clsMap) {
    StringBuilder sb = new StringBuilder();
    int count = 0;
    for (Instruction linst : inst) {
        // check that only cp instruction are transmitted
        if (!(linst instanceof CPInstruction || linst instanceof ExternalFunctionInvocationInstruction))
            throw new DMLRuntimeException(NOT_SUPPORTED_MR_INSTRUCTION + " " + linst.getClass().getName() + "\n" + linst);
        // obtain serialized version of generated classes
        if (linst instanceof SpoofCPInstruction) {
            Class<?> cla = ((SpoofCPInstruction) linst).getOperatorClass();
            clsMap.put(cla.getName(), CodegenUtils.getClassData(cla.getName()));
        }
        if (count > 0)
            sb.append(ELEMENT_DELIM);
        sb.append(checkAndReplaceLiterals(linst.toString()));
        count++;
    }
    return sb.toString();
}

Example 23 with CPInstruction

use of org.apache.sysml.runtime.instructions.cp.CPInstruction in project systemml by apache.

the class Dag method generateInstructionsForInputVariables.

/**
 * Method to generate createvar instructions, which creates a new entry
 * in the symbol table. One instruction is generated for every LOP that is
 * 1) type Data and
 * 2) persistent and
 * 3) matrix and
 * 4) read
 *
 * Transient reads needn't be considered here since the previous program
 * block would already create appropriate entries in the symbol table.
 *
 * @param nodes_v list of nodes
 * @param inst list of instructions
 */
private static void generateInstructionsForInputVariables(ArrayList<Lop> nodes_v, ArrayList<Instruction> inst) {
    for (Lop n : nodes_v) {
        if (n.getExecLocation() == ExecLocation.Data && !((Data) n).isTransient() && ((Data) n).getOperationType() == OperationTypes.READ && (n.getDataType() == DataType.MATRIX || n.getDataType() == DataType.FRAME)) {
            if (!((Data) n).isLiteral()) {
                try {
                    String inst_string = n.getInstructions();
                    CPInstruction currInstr = CPInstructionParser.parseSingleInstruction(inst_string);
                    currInstr.setLocation(n);
                    inst.add(currInstr);
                } catch (DMLRuntimeException e) {
                    throw new LopsException(n.printErrorLocation() + "error generating instructions from input variables in Dag -- \n", e);
                }
            }
        }
    }
}

Example 24 with CPInstruction

use of org.apache.sysml.runtime.instructions.cp.CPInstruction in project systemml by apache.

the class Dag method setupNodeOutputs.

/**
 * Method to setup output filenames and outputInfos, and to generate related instructions
 *
 * @param node low-level operator
 * @param et exec type
 * @param cellModeOverride override mode
 * @param copyTWrite ?
 * @return node output
 */
private NodeOutput setupNodeOutputs(Lop node, ExecType et, boolean cellModeOverride, boolean copyTWrite) {
    OutputParameters oparams = node.getOutputParameters();
    NodeOutput out = new NodeOutput();
    node.setConsumerCount(node.getOutputs().size());
    // Compute the output format for this node
    out.setOutInfo(getOutputInfo(node, cellModeOverride));
    // since outputs are explicitly specified
    if (node.getExecLocation() != ExecLocation.Data) {
        if (node.getDataType() == DataType.SCALAR) {
            oparams.setLabel(Lop.SCALAR_VAR_NAME_PREFIX + var_index.getNextID());
            out.setVarName(oparams.getLabel());
            Instruction currInstr = VariableCPInstruction.prepareRemoveInstruction(oparams.getLabel());
            currInstr.setLocation(node);
            out.addLastInstruction(currInstr);
        } else if (// general case
        !(node instanceof FunctionCallCP)) {
            // generate temporary filename and a variable name to hold the
            // output produced by "rootNode"
            oparams.setFile_name(getNextUniqueFilename());
            oparams.setLabel(getNextUniqueVarname(node.getDataType()));
            // generate an instruction that creates a symbol table entry for the new variable
            // String createInst = prepareVariableInstruction("createvar", node);
            // out.addPreInstruction(CPInstructionParser.parseSingleInstruction(createInst));
            int rpb = (int) oparams.getRowsInBlock();
            int cpb = (int) oparams.getColsInBlock();
            Instruction createvarInst = VariableCPInstruction.prepareCreateVariableInstruction(oparams.getLabel(), oparams.getFile_name(), true, node.getDataType(), OutputInfo.outputInfoToString(getOutputInfo(node, false)), new MatrixCharacteristics(oparams.getNumRows(), oparams.getNumCols(), rpb, cpb, oparams.getNnz()), oparams.getUpdateType());
            createvarInst.setLocation(node);
            out.addPreInstruction(createvarInst);
            // temp file as well as the variable has to be deleted at the end
            Instruction currInstr = VariableCPInstruction.prepareRemoveInstruction(oparams.getLabel());
            currInstr.setLocation(node);
            out.addLastInstruction(currInstr);
            // finally, add the generated filename and variable name to the list of outputs
            out.setFileName(oparams.getFile_name());
            out.setVarName(oparams.getLabel());
        } else {
            // If the function call is set with output lops (e.g., multi return builtin),
            // generate a createvar instruction for each function output
            FunctionCallCP fcall = (FunctionCallCP) node;
            if (fcall.getFunctionOutputs() != null) {
                for (Lop fnOut : fcall.getFunctionOutputs()) {
                    OutputParameters fnOutParams = fnOut.getOutputParameters();
                    // OutputInfo oinfo = getOutputInfo((N)fnOut, false);
                    Instruction createvarInst = VariableCPInstruction.prepareCreateVariableInstruction(fnOutParams.getLabel(), getFilePath() + fnOutParams.getLabel(), true, fnOut.getDataType(), OutputInfo.outputInfoToString(getOutputInfo(fnOut, false)), new MatrixCharacteristics(fnOutParams.getNumRows(), fnOutParams.getNumCols(), (int) fnOutParams.getRowsInBlock(), (int) fnOutParams.getColsInBlock(), fnOutParams.getNnz()), oparams.getUpdateType());
                    if (node._beginLine != 0)
                        createvarInst.setLocation(node);
                    else
                        createvarInst.setLocation(fnOut);
                    out.addPreInstruction(createvarInst);
                }
            }
        }
    } else // rootNode is of type Data
    {
        if (node.getDataType() == DataType.SCALAR) {
            // generate assignment operations for final and transient writes
            if (oparams.getFile_name() == null && !(node instanceof Data && ((Data) node).isPersistentWrite())) {
                String io_inst = prepareAssignVarInstruction(node.getInputs().get(0), node);
                CPInstruction currInstr = CPInstructionParser.parseSingleInstruction(io_inst);
                if (node._beginLine != 0)
                    currInstr.setLocation(node);
                else if (!node.getInputs().isEmpty())
                    currInstr.setLocation(node.getInputs().get(0));
                out.addLastInstruction(currInstr);
            } else {
                // CP PERSISTENT WRITE SCALARS
                Lop fname = ((Data) node).getNamedInputLop(DataExpression.IO_FILENAME);
                String io_inst = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), fname.getOutputParameters().getLabel());
                CPInstruction currInstr = CPInstructionParser.parseSingleInstruction(io_inst);
                if (node._beginLine != 0)
                    currInstr.setLocation(node);
                else if (!node.getInputs().isEmpty())
                    currInstr.setLocation(node.getInputs().get(0));
                out.addLastInstruction(currInstr);
            }
        } else {
            if (((Data) node).isTransient()) {
                if (et == ExecType.CP) {
                    // If transient matrix write is in CP then its input MUST be executed in CP as well.
                    // get variable and filename associated with the input
                    String inputFileName = node.getInputs().get(0).getOutputParameters().getFile_name();
                    String inputVarName = node.getInputs().get(0).getOutputParameters().getLabel();
                    String constVarName = oparams.getLabel();
                    String constFileName = inputFileName + constVarName;
                    /*
						 * Symbol Table state must change as follows:
						 * 
						 * FROM:
						 *     mvar1 -> temp21
						 *  
						 * TO:
						 *     mVar1 -> temp21
						 *     tVarH -> temp21
						 */
                    Instruction currInstr = VariableCPInstruction.prepareCopyInstruction(inputVarName, constVarName);
                    currInstr.setLocation(node);
                    out.addLastInstruction(currInstr);
                    out.setFileName(constFileName);
                } else {
                    if (copyTWrite) {
                        Instruction currInstr = VariableCPInstruction.prepareCopyInstruction(node.getInputs().get(0).getOutputParameters().getLabel(), oparams.getLabel());
                        currInstr.setLocation(node);
                        out.addLastInstruction(currInstr);
                        return out;
                    }
                    /*
						 * Since the "rootNode" is a transient data node, we first need to generate a 
						 * temporary filename as well as a variable name to hold the <i>immediate</i> 
						 * output produced by "rootNode". These generated HDFS filename and the 
						 * variable name must be changed at the end of an iteration/program block 
						 * so that the subsequent iteration/program block can correctly access the 
						 * generated data. Therefore, we need to distinguish between the following:
						 * 
						 *   1) Temporary file name & variable name: They hold the immediate output 
						 *   produced by "rootNode". Both names are generated below.
						 *   
						 *   2) Constant file name & variable name: They are constant across iterations. 
						 *   Variable name is given by rootNode's label that is created in the upper layers.  
						 *   File name is generated by concatenating "temporary file name" and "constant variable name".
						 *   
						 * Temporary files must be moved to constant files at the end of the iteration/program block.
						 */
                    // generate temporary filename & var name
                    String tempVarName = oparams.getLabel() + "temp";
                    String tempFileName = getNextUniqueFilename();
                    int rpb = (int) oparams.getRowsInBlock();
                    int cpb = (int) oparams.getColsInBlock();
                    Instruction createvarInst = VariableCPInstruction.prepareCreateVariableInstruction(tempVarName, tempFileName, true, node.getDataType(), OutputInfo.outputInfoToString(out.getOutInfo()), new MatrixCharacteristics(oparams.getNumRows(), oparams.getNumCols(), rpb, cpb, oparams.getNnz()), oparams.getUpdateType());
                    createvarInst.setLocation(node);
                    out.addPreInstruction(createvarInst);
                    String constVarName = oparams.getLabel();
                    String constFileName = tempFileName + constVarName;
                    oparams.setFile_name(getFilePath() + constFileName);
                    /*
						 * Since this is a node that denotes a transient read/write, we need to make sure 
						 * that the data computed for a given variable in a given iteration is passed on 
						 * to the next iteration. This is done by generating miscellaneous instructions 
						 * that gets executed at the end of the program block.
						 * 
						 * The state of the symbol table must change 
						 * 
						 * FROM: 
						 *     tVarA -> temp21tVarA (old copy of temp21)
						 *     tVarAtemp -> temp21  (new copy that should override the old copy) 
						 *
						 * TO:
						 *     tVarA -> temp21tVarA
						 */
                    // Generate a single mvvar instruction (e.g., mvvar tempA A)
                    // instead of two instructions "cpvar tempA A" and "rmvar tempA"
                    Instruction currInstr = VariableCPInstruction.prepareMoveInstruction(tempVarName, constVarName);
                    currInstr.setLocation(node);
                    out.addLastInstruction(currInstr);
                    // finally, add the temporary filename and variable name to the list of outputs
                    out.setFileName(tempFileName);
                    out.setVarName(tempVarName);
                }
            } else // rootNode is not a transient write. It is a persistent write.
            {
                if (et == ExecType.MR) {
                    // MR PERSISTENT WRITE
                    // create a variable to hold the result produced by this "rootNode"
                    oparams.setLabel("pVar" + var_index.getNextID());
                    int rpb = (int) oparams.getRowsInBlock();
                    int cpb = (int) oparams.getColsInBlock();
                    Lop fnameLop = ((Data) node).getNamedInputLop(DataExpression.IO_FILENAME);
                    String fnameStr = (fnameLop instanceof Data && ((Data) fnameLop).isLiteral()) ? fnameLop.getOutputParameters().getLabel() : Lop.VARIABLE_NAME_PLACEHOLDER + fnameLop.getOutputParameters().getLabel() + Lop.VARIABLE_NAME_PLACEHOLDER;
                    Instruction createvarInst;
                    // part MM format file on hdfs.
                    if (oparams.getFormat() == Format.CSV) {
                        String tempFileName = getNextUniqueFilename();
                        String createInst = node.getInstructions(tempFileName);
                        createvarInst = CPInstructionParser.parseSingleInstruction(createInst);
                        // NOTE: no instruction patching because final write from cp instruction
                        String writeInst = node.getInstructions(oparams.getLabel(), fnameLop.getOutputParameters().getLabel());
                        CPInstruction currInstr = CPInstructionParser.parseSingleInstruction(writeInst);
                        currInstr.setLocation(node);
                        out.addPostInstruction(currInstr);
                        // remove the variable
                        CPInstruction tempInstr = CPInstructionParser.parseSingleInstruction("CP" + Lop.OPERAND_DELIMITOR + "rmfilevar" + Lop.OPERAND_DELIMITOR + oparams.getLabel() + Lop.VALUETYPE_PREFIX + Expression.ValueType.UNKNOWN + Lop.OPERAND_DELIMITOR + "true" + Lop.VALUETYPE_PREFIX + "BOOLEAN");
                        tempInstr.setLocation(node);
                        out.addLastInstruction(tempInstr);
                    } else if (oparams.getFormat() == Format.MM) {
                        createvarInst = VariableCPInstruction.prepareCreateVariableInstruction(oparams.getLabel(), getNextUniqueFilename(), false, node.getDataType(), OutputInfo.outputInfoToString(getOutputInfo(node, false)), new MatrixCharacteristics(oparams.getNumRows(), oparams.getNumCols(), rpb, cpb, oparams.getNnz()), oparams.getUpdateType());
                        // NOTE: no instruction patching because final write from cp instruction
                        String writeInst = node.getInstructions(oparams.getLabel(), fnameLop.getOutputParameters().getLabel());
                        CPInstruction currInstr = CPInstructionParser.parseSingleInstruction(writeInst);
                        currInstr.setLocation(node);
                        out.addPostInstruction(currInstr);
                        // remove the variable
                        CPInstruction tempInstr = CPInstructionParser.parseSingleInstruction("CP" + Lop.OPERAND_DELIMITOR + "rmfilevar" + Lop.OPERAND_DELIMITOR + oparams.getLabel() + Lop.VALUETYPE_PREFIX + Expression.ValueType.UNKNOWN + Lop.OPERAND_DELIMITOR + "true" + Lop.VALUETYPE_PREFIX + "BOOLEAN");
                        tempInstr.setLocation(node);
                        out.addLastInstruction(tempInstr);
                    } else {
                        createvarInst = VariableCPInstruction.prepareCreateVariableInstruction(oparams.getLabel(), fnameStr, false, node.getDataType(), OutputInfo.outputInfoToString(getOutputInfo(node, false)), new MatrixCharacteristics(oparams.getNumRows(), oparams.getNumCols(), rpb, cpb, oparams.getNnz()), oparams.getUpdateType());
                        // remove the variable
                        CPInstruction currInstr = CPInstructionParser.parseSingleInstruction("CP" + Lop.OPERAND_DELIMITOR + "rmfilevar" + Lop.OPERAND_DELIMITOR + oparams.getLabel() + Lop.VALUETYPE_PREFIX + Expression.ValueType.UNKNOWN + Lop.OPERAND_DELIMITOR + "false" + Lop.VALUETYPE_PREFIX + "BOOLEAN");
                        currInstr.setLocation(node);
                        out.addLastInstruction(currInstr);
                    }
                    createvarInst.setLocation(node);
                    out.addPreInstruction(createvarInst);
                    // finally, add the filename and variable name to the list of outputs
                    out.setFileName(oparams.getFile_name());
                    out.setVarName(oparams.getLabel());
                } else {
                    // CP PERSISTENT WRITE
                    // generate a write instruction that writes matrix to HDFS
                    Lop fname = ((Data) node).getNamedInputLop(DataExpression.IO_FILENAME);
                    String io_inst = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), fname.getOutputParameters().getLabel());
                    Instruction currInstr = (node.getExecType() == ExecType.SPARK) ? SPInstructionParser.parseSingleInstruction(io_inst) : CPInstructionParser.parseSingleInstruction(io_inst);
                    currInstr.setLocation((!node.getInputs().isEmpty() && node.getInputs().get(0)._beginLine != 0) ? node.getInputs().get(0) : node);
                    out.addLastInstruction(currInstr);
                }
            }
        }
    }
    return out;
}