Examples with Instruction org.apache.sysml.runtime.instructions.Instruction used on opensource projects

Example 46 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project incubator-systemml by apache.

the class Dag method generateMapReduceInstructions.

/**
 * Method to generate MapReduce job instructions from a given set of nodes.
 *
 * @param execNodes list of exec nodes
 * @param inst list of instructions
 * @param writeinst list of write instructions
 * @param deleteinst list of delete instructions
 * @param rmvarinst list of rmvar instructions
 * @param jt job type
 */
private void generateMapReduceInstructions(ArrayList<Lop> execNodes, ArrayList<Instruction> inst, ArrayList<Instruction> writeinst, ArrayList<Instruction> deleteinst, ArrayList<Instruction> rmvarinst, JobType jt) {
    ArrayList<Byte> resultIndices = new ArrayList<>();
    ArrayList<String> inputs = new ArrayList<>();
    ArrayList<String> outputs = new ArrayList<>();
    ArrayList<InputInfo> inputInfos = new ArrayList<>();
    ArrayList<OutputInfo> outputInfos = new ArrayList<>();
    ArrayList<Long> numRows = new ArrayList<>();
    ArrayList<Long> numCols = new ArrayList<>();
    ArrayList<Long> numRowsPerBlock = new ArrayList<>();
    ArrayList<Long> numColsPerBlock = new ArrayList<>();
    ArrayList<String> mapperInstructions = new ArrayList<>();
    ArrayList<String> randInstructions = new ArrayList<>();
    ArrayList<String> recordReaderInstructions = new ArrayList<>();
    int numReducers = 0;
    int replication = 1;
    ArrayList<String> inputLabels = new ArrayList<>();
    ArrayList<String> outputLabels = new ArrayList<>();
    ArrayList<Instruction> renameInstructions = new ArrayList<>();
    ArrayList<Instruction> variableInstructions = new ArrayList<>();
    ArrayList<Instruction> postInstructions = new ArrayList<>();
    ArrayList<Integer> MRJobLineNumbers = null;
    if (DMLScript.ENABLE_DEBUG_MODE) {
        MRJobLineNumbers = new ArrayList<>();
    }
    ArrayList<Lop> inputLops = new ArrayList<>();
    boolean cellModeOverride = false;
    /* Find the nodes that produce an output */
    ArrayList<Lop> rootNodes = new ArrayList<>();
    getOutputNodes(execNodes, rootNodes, jt);
    if (LOG.isTraceEnabled())
        LOG.trace("# of root nodes = " + rootNodes.size());
    /* Remove transient writes that are simple copy of transient reads */
    if (jt == JobType.GMR || jt == JobType.GMRCELL) {
        ArrayList<Lop> markedNodes = new ArrayList<>();
        // only keep data nodes that are results of some computation.
        for (Lop rnode : rootNodes) {
            if (rnode.getExecLocation() == ExecLocation.Data && ((Data) rnode).isTransient() && ((Data) rnode).getOperationType() == OperationTypes.WRITE && ((Data) rnode).getDataType() == DataType.MATRIX) {
                // no computation, just a copy
                if (rnode.getInputs().get(0).getExecLocation() == ExecLocation.Data && ((Data) rnode.getInputs().get(0)).isTransient() && rnode.getOutputParameters().getLabel().equals(rnode.getInputs().get(0).getOutputParameters().getLabel())) {
                    markedNodes.add(rnode);
                }
            }
        }
        // delete marked nodes
        rootNodes.removeAll(markedNodes);
        markedNodes.clear();
        if (rootNodes.isEmpty())
            return;
    }
    // structure that maps node to their indices that will be used in the instructions
    HashMap<Lop, Integer> nodeIndexMapping = new HashMap<>();
    for (Lop rnode : rootNodes) {
        getInputPathsAndParameters(rnode, execNodes, inputs, inputInfos, numRows, numCols, numRowsPerBlock, numColsPerBlock, nodeIndexMapping, inputLabels, inputLops, MRJobLineNumbers);
    }
    // In case of RAND job, instructions are defined in the input file
    if (jt == JobType.DATAGEN)
        randInstructions = inputs;
    int[] start_index = new int[1];
    start_index[0] = inputs.size();
    // currently, recordreader instructions are allowed only in GMR jobs
    if (jt == JobType.GMR || jt == JobType.GMRCELL) {
        for (Lop rnode : rootNodes) {
            getRecordReaderInstructions(rnode, execNodes, inputs, recordReaderInstructions, nodeIndexMapping, start_index, inputLabels, inputLops, MRJobLineNumbers);
            if (recordReaderInstructions.size() > 1)
                throw new LopsException("MapReduce job can only have a single recordreader instruction: " + recordReaderInstructions.toString());
        }
    }
    // 
    if (jt != JobType.REBLOCK && jt != JobType.CSV_REBLOCK && jt != JobType.DATAGEN) {
        for (int i = 0; i < inputInfos.size(); i++) if (inputInfos.get(i) == InputInfo.BinaryCellInputInfo || inputInfos.get(i) == InputInfo.TextCellInputInfo)
            cellModeOverride = true;
    }
    if (!recordReaderInstructions.isEmpty() || jt == JobType.GROUPED_AGG)
        cellModeOverride = true;
    for (int i = 0; i < rootNodes.size(); i++) {
        getMapperInstructions(rootNodes.get(i), execNodes, inputs, mapperInstructions, nodeIndexMapping, start_index, inputLabels, inputLops, MRJobLineNumbers);
    }
    if (LOG.isTraceEnabled()) {
        LOG.trace("    Input strings: " + inputs.toString());
        if (jt == JobType.DATAGEN)
            LOG.trace("    Rand instructions: " + getCSVString(randInstructions));
        if (jt == JobType.GMR)
            LOG.trace("    RecordReader instructions: " + getCSVString(recordReaderInstructions));
        LOG.trace("    Mapper instructions: " + getCSVString(mapperInstructions));
    }
    /* Get Shuffle and Reducer Instructions */
    ArrayList<String> shuffleInstructions = new ArrayList<>();
    ArrayList<String> aggInstructionsReducer = new ArrayList<>();
    ArrayList<String> otherInstructionsReducer = new ArrayList<>();
    for (Lop rn : rootNodes) {
        int resultIndex = getAggAndOtherInstructions(rn, execNodes, shuffleInstructions, aggInstructionsReducer, otherInstructionsReducer, nodeIndexMapping, start_index, inputLabels, inputLops, MRJobLineNumbers);
        if (resultIndex == -1)
            throw new LopsException("Unexpected error in piggybacking!");
        if (rn.getExecLocation() == ExecLocation.Data && ((Data) rn).getOperationType() == Data.OperationTypes.WRITE && ((Data) rn).isTransient() && rootNodes.contains(rn.getInputs().get(0))) {
            // Both rn (a transient write) and its input are root nodes.
            // Instead of creating two copies of the data, simply generate a cpvar instruction
            NodeOutput out = setupNodeOutputs(rn, ExecType.MR, cellModeOverride, true);
            writeinst.addAll(out.getLastInstructions());
        } else {
            resultIndices.add(Byte.valueOf((byte) resultIndex));
            // setup output filenames and outputInfos and generate related instructions
            NodeOutput out = setupNodeOutputs(rn, ExecType.MR, cellModeOverride, false);
            outputLabels.add(out.getVarName());
            outputs.add(out.getFileName());
            outputInfos.add(out.getOutInfo());
            if (LOG.isTraceEnabled()) {
                LOG.trace("    Output Info: " + out.getFileName() + ";" + OutputInfo.outputInfoToString(out.getOutInfo()) + ";" + out.getVarName());
            }
            renameInstructions.addAll(out.getLastInstructions());
            variableInstructions.addAll(out.getPreInstructions());
            postInstructions.addAll(out.getPostInstructions());
        }
    }
    /* Determine if the output dimensions are known */
    byte[] resultIndicesByte = new byte[resultIndices.size()];
    for (int i = 0; i < resultIndicesByte.length; i++) {
        resultIndicesByte[i] = resultIndices.get(i).byteValue();
    }
    if (LOG.isTraceEnabled()) {
        LOG.trace("    Shuffle Instructions: " + getCSVString(shuffleInstructions));
        LOG.trace("    Aggregate Instructions: " + getCSVString(aggInstructionsReducer));
        LOG.trace("    Other instructions =" + getCSVString(otherInstructionsReducer));
        LOG.trace("    Output strings: " + outputs.toString());
        LOG.trace("    ResultIndices = " + resultIndices.toString());
    }
    /* Prepare the MapReduce job instruction */
    MRJobInstruction mr = new MRJobInstruction(jt);
    // check if this is a map-only job. If not, set the number of reducers
    if (!shuffleInstructions.isEmpty() || !aggInstructionsReducer.isEmpty() || !otherInstructionsReducer.isEmpty())
        numReducers = total_reducers;
    // set inputs, outputs, and other other properties for the job
    mr.setInputOutputLabels(inputLabels.toArray(new String[0]), outputLabels.toArray(new String[0]));
    mr.setOutputs(resultIndicesByte);
    mr.setDimsUnknownFilePrefix(getFilePath());
    mr.setNumberOfReducers(numReducers);
    mr.setReplication(replication);
    // set instructions for recordReader and mapper
    mr.setRecordReaderInstructions(getCSVString(recordReaderInstructions));
    mr.setMapperInstructions(getCSVString(mapperInstructions));
    // compute and set mapper memory requirements (for consistency of runtime piggybacking)
    if (jt == JobType.GMR) {
        double mem = 0;
        for (Lop n : execNodes) mem += computeFootprintInMapper(n);
        mr.setMemoryRequirements(mem);
    }
    if (jt == JobType.DATAGEN)
        mr.setRandInstructions(getCSVString(randInstructions));
    // set shuffle instructions
    mr.setShuffleInstructions(getCSVString(shuffleInstructions));
    // set reducer instruction
    mr.setAggregateInstructionsInReducer(getCSVString(aggInstructionsReducer));
    mr.setOtherInstructionsInReducer(getCSVString(otherInstructionsReducer));
    if (DMLScript.ENABLE_DEBUG_MODE) {
        // set line number information for each MR instruction
        mr.setMRJobInstructionsLineNumbers(MRJobLineNumbers);
    }
    /* Add the prepared instructions to output set */
    inst.addAll(variableInstructions);
    inst.add(mr);
    inst.addAll(postInstructions);
    deleteinst.addAll(renameInstructions);
    for (Lop l : inputLops) {
        if (DMLScript.ENABLE_DEBUG_MODE) {
            processConsumers(l, rmvarinst, deleteinst, l);
        } else {
            processConsumers(l, rmvarinst, deleteinst, null);
        }
    }
}

Example 47 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project incubator-systemml by apache.

the class Dag method processConsumers.

private static void processConsumers(Lop node, ArrayList<Instruction> inst, ArrayList<Instruction> deleteInst, Lop locationInfo) {
    // if the count becomes zero, then then variable associated w/ input can be removed
    if (node.removeConsumer() == 0) {
        if (node.getExecLocation() == ExecLocation.Data && ((Data) node).isLiteral()) {
            return;
        }
        String label = node.getOutputParameters().getLabel();
        Instruction currInstr = VariableCPInstruction.prepareRemoveInstruction(label);
        if (locationInfo != null)
            currInstr.setLocation(locationInfo);
        else
            currInstr.setLocation(node);
        inst.add(currInstr);
        excludeRemoveInstruction(label, deleteInst);
    }
}

Example 48 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project incubator-systemml by apache.

the class DMLTranslator method createRuntimeProgramBlock.

public ProgramBlock createRuntimeProgramBlock(Program prog, StatementBlock sb, DMLConfig config) {
    Dag<Lop> dag = null;
    Dag<Lop> pred_dag = null;
    ArrayList<Instruction> instruct;
    ArrayList<Instruction> pred_instruct = null;
    ProgramBlock retPB = null;
    // process While Statement - add runtime program blocks to program
    if (sb instanceof WhileStatementBlock) {
        // create DAG for loop predicates
        pred_dag = new Dag<>();
        ((WhileStatementBlock) sb).get_predicateLops().addToDag(pred_dag);
        // create instructions for loop predicates
        pred_instruct = new ArrayList<>();
        ArrayList<Instruction> pInst = pred_dag.getJobs(null, config);
        for (Instruction i : pInst) {
            pred_instruct.add(i);
        }
        // create while program block
        WhileProgramBlock rtpb = new WhileProgramBlock(prog, pred_instruct);
        // // process the body of the while statement block ////
        WhileStatementBlock wsb = (WhileStatementBlock) sb;
        WhileStatement wstmt = (WhileStatement) wsb.getStatement(0);
        for (StatementBlock sblock : wstmt.getBody()) {
            // process the body
            ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
            rtpb.addProgramBlock(childBlock);
        }
        retPB = rtpb;
        // add statement block
        retPB.setStatementBlock(sb);
        // add location information
        retPB.setParseInfo(sb);
    } else // process If Statement - add runtime program blocks to program
    if (sb instanceof IfStatementBlock) {
        // create DAG for loop predicates
        pred_dag = new Dag<>();
        ((IfStatementBlock) sb).get_predicateLops().addToDag(pred_dag);
        // create instructions for loop predicates
        pred_instruct = new ArrayList<>();
        ArrayList<Instruction> pInst = pred_dag.getJobs(null, config);
        for (Instruction i : pInst) {
            pred_instruct.add(i);
        }
        // create if program block
        IfProgramBlock rtpb = new IfProgramBlock(prog, pred_instruct);
        // process the body of the if statement block
        IfStatementBlock isb = (IfStatementBlock) sb;
        IfStatement istmt = (IfStatement) isb.getStatement(0);
        // process the if body
        for (StatementBlock sblock : istmt.getIfBody()) {
            ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
            rtpb.addProgramBlockIfBody(childBlock);
        }
        // process the else body
        for (StatementBlock sblock : istmt.getElseBody()) {
            ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
            rtpb.addProgramBlockElseBody(childBlock);
        }
        retPB = rtpb;
        // post processing for generating missing instructions
        // retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
        // add statement block
        retPB.setStatementBlock(sb);
        // add location information
        retPB.setParseInfo(sb);
    } else // NOTE: applies to ForStatementBlock and ParForStatementBlock
    if (sb instanceof ForStatementBlock) {
        ForStatementBlock fsb = (ForStatementBlock) sb;
        // create DAGs for loop predicates
        Dag<Lop> fromDag = new Dag<>();
        Dag<Lop> toDag = new Dag<>();
        Dag<Lop> incrementDag = new Dag<>();
        if (fsb.getFromHops() != null)
            fsb.getFromLops().addToDag(fromDag);
        if (fsb.getToHops() != null)
            fsb.getToLops().addToDag(toDag);
        if (fsb.getIncrementHops() != null)
            fsb.getIncrementLops().addToDag(incrementDag);
        // create instructions for loop predicates
        ArrayList<Instruction> fromInstructions = fromDag.getJobs(null, config);
        ArrayList<Instruction> toInstructions = toDag.getJobs(null, config);
        ArrayList<Instruction> incrementInstructions = incrementDag.getJobs(null, config);
        // create for program block
        ForProgramBlock rtpb = null;
        IterablePredicate iterPred = fsb.getIterPredicate();
        if (sb instanceof ParForStatementBlock && ConfigurationManager.isParallelParFor()) {
            rtpb = new ParForProgramBlock(prog, iterPred.getIterVar().getName(), iterPred.getParForParams(), ((ParForStatementBlock) sb).getResultVariables());
            ParForProgramBlock pfrtpb = (ParForProgramBlock) rtpb;
            // used for optimization and creating unscoped variables
            pfrtpb.setStatementBlock((ParForStatementBlock) sb);
        } else {
            // ForStatementBlock
            rtpb = new ForProgramBlock(prog, iterPred.getIterVar().getName());
        }
        rtpb.setFromInstructions(fromInstructions);
        rtpb.setToInstructions(toInstructions);
        rtpb.setIncrementInstructions(incrementInstructions);
        // process the body of the for statement block
        ForStatement fs = (ForStatement) fsb.getStatement(0);
        for (StatementBlock sblock : fs.getBody()) {
            ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
            rtpb.addProgramBlock(childBlock);
        }
        retPB = rtpb;
        // add statement block
        retPB.setStatementBlock(sb);
        // add location information
        retPB.setParseInfo(sb);
    } else // process function statement block - add runtime program blocks to program
    if (sb instanceof FunctionStatementBlock) {
        FunctionStatementBlock fsb = (FunctionStatementBlock) sb;
        FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0);
        FunctionProgramBlock rtpb = null;
        if (fstmt instanceof ExternalFunctionStatement) {
            // create external function program block
            String execType = ((ExternalFunctionStatement) fstmt).getOtherParams().get(ExternalFunctionStatement.EXEC_TYPE);
            boolean isCP = (execType.equals(ExternalFunctionStatement.IN_MEMORY)) ? true : false;
            StringBuilder buff = new StringBuilder();
            buff.append(config.getTextValue(DMLConfig.SCRATCH_SPACE));
            buff.append(Lop.FILE_SEPARATOR);
            buff.append(Lop.PROCESS_PREFIX);
            buff.append(DMLScript.getUUID());
            buff.append(Lop.FILE_SEPARATOR);
            buff.append(ProgramConverter.CP_ROOT_THREAD_ID);
            buff.append(Lop.FILE_SEPARATOR);
            buff.append("PackageSupport");
            buff.append(Lop.FILE_SEPARATOR);
            String basedir = buff.toString();
            if (isCP) {
                rtpb = new ExternalFunctionProgramBlockCP(prog, fstmt.getInputParams(), fstmt.getOutputParams(), ((ExternalFunctionStatement) fstmt).getOtherParams(), basedir);
            } else {
                rtpb = new ExternalFunctionProgramBlock(prog, fstmt.getInputParams(), fstmt.getOutputParams(), ((ExternalFunctionStatement) fstmt).getOtherParams(), basedir);
            }
            if (!fstmt.getBody().isEmpty()) {
                LOG.error(fstmt.printErrorLocation() + "ExternalFunctionStatementBlock should have no statement blocks in body");
                throw new LopsException(fstmt.printErrorLocation() + "ExternalFunctionStatementBlock should have no statement blocks in body");
            }
        } else {
            // create function program block
            rtpb = new FunctionProgramBlock(prog, fstmt.getInputParams(), fstmt.getOutputParams());
            // process the function statement body
            for (StatementBlock sblock : fstmt.getBody()) {
                // process the body
                ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
                rtpb.addProgramBlock(childBlock);
            }
        }
        // check there are actually Lops in to process (loop stmt body will not have any)
        if (fsb.getLops() != null && !fsb.getLops().isEmpty()) {
            LOG.error(fsb.printBlockErrorLocation() + "FunctionStatementBlock should have no Lops");
            throw new LopsException(fsb.printBlockErrorLocation() + "FunctionStatementBlock should have no Lops");
        }
        retPB = rtpb;
        // add location information
        retPB.setParseInfo(sb);
    } else {
        // handle general case
        ProgramBlock rtpb = new ProgramBlock(prog);
        // DAGs for Lops
        dag = new Dag<>();
        // check there are actually Lops in to process (loop stmt body will not have any)
        if (sb.getLops() != null && !sb.getLops().isEmpty()) {
            for (Lop l : sb.getLops()) {
                l.addToDag(dag);
            }
            // Instructions for Lops DAGs
            instruct = dag.getJobs(sb, config);
            rtpb.addInstructions(instruct);
        }
        retPB = rtpb;
        // post processing for generating missing instructions
        // retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
        // add statement block
        retPB.setStatementBlock(sb);
        // add location information
        retPB.setParseInfo(sb);
    }
    return retPB;
}

Example 49 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project incubator-systemml by apache.

the class MLContextUtil method deleteRemoveVariableInstructions.

/**
 * Recursively traverse program block to delete 'remove variable'
 * instructions.
 *
 * @param pb
 *            Program block
 */
private static void deleteRemoveVariableInstructions(ProgramBlock pb) {
    if (pb instanceof WhileProgramBlock) {
        WhileProgramBlock wpb = (WhileProgramBlock) pb;
        for (ProgramBlock pbc : wpb.getChildBlocks()) deleteRemoveVariableInstructions(pbc);
    } else if (pb instanceof IfProgramBlock) {
        IfProgramBlock ipb = (IfProgramBlock) pb;
        for (ProgramBlock pbc : ipb.getChildBlocksIfBody()) deleteRemoveVariableInstructions(pbc);
        for (ProgramBlock pbc : ipb.getChildBlocksElseBody()) deleteRemoveVariableInstructions(pbc);
    } else if (pb instanceof ForProgramBlock) {
        ForProgramBlock fpb = (ForProgramBlock) pb;
        for (ProgramBlock pbc : fpb.getChildBlocks()) deleteRemoveVariableInstructions(pbc);
    } else {
        ArrayList<Instruction> instructions = pb.getInstructions();
        deleteRemoveVariableInstructions(instructions);
    }
}

Example 50 with Instruction

use of org.apache.sysml.runtime.instructions.Instruction in project incubator-systemml by apache.

the class DMLDebuggerProgramInfo method accesBreakpointInstruction.

/**
 * Access breakpoint instruction at specified line number in set of instructions (if valid)
 * @param instructions Instructions for current program block
 * @param lineNumber Location for inserting breakpoint
 * @param op Breakpoint operation
 * @param status Current breakpoint status
 */
private void accesBreakpointInstruction(ArrayList<Instruction> instructions, int lineNumber, int op, BPINSTRUCTION_STATUS status) {
    for (int i = 0; i < instructions.size(); i++) {
        Instruction currInst = instructions.get(i);
        if (op == 0) {
            if (currInst instanceof MRJobInstruction) {
                MRJobInstruction currMRInst = (MRJobInstruction) currInst;
                // Check if current instruction line number correspond to breakpoint line number
                if (currMRInst.findMRInstructions(lineNumber)) {
                    BreakPointInstruction breakpoint = new BreakPointInstruction();
                    breakpoint.setLocation(currInst);
                    breakpoint.setInstID(instID++);
                    breakpoint.setBPInstructionLocation(location);
                    instructions.add(i, breakpoint);
                    DMLBreakpointManager.insertBreakpoint(breakpoint, lineNumber);
                    return;
                }
            } else if (currInst instanceof CPInstruction || currInst instanceof SPInstruction) {
                // Check if current instruction line number correspond to breakpoint line number
                if (currInst.getLineNum() == lineNumber) {
                    BreakPointInstruction breakpoint = new BreakPointInstruction();
                    breakpoint.setLocation(currInst);
                    breakpoint.setInstID(instID++);
                    breakpoint.setBPInstructionLocation(location);
                    instructions.add(i, breakpoint);
                    DMLBreakpointManager.insertBreakpoint(breakpoint, lineNumber);
                    return;
                }
            } else if (currInst instanceof BreakPointInstruction && currInst.getLineNum() == lineNumber) {
                BreakPointInstruction breakpoint = (BreakPointInstruction) currInst;
                breakpoint.setBPInstructionStatus(BPINSTRUCTION_STATUS.ENABLED);
                breakpoint.setBPInstructionLocation(location);
                instructions.set(i, breakpoint);
                DMLBreakpointManager.updateBreakpoint(lineNumber, status);
                return;
            }
        } else {
            // Check if current instruction line number correspond to breakpoint line number
            if (currInst instanceof BreakPointInstruction && currInst.getLineNum() == lineNumber) {
                if (op == 1) {
                    BreakPointInstruction breakpoint = (BreakPointInstruction) currInst;
                    breakpoint.setLocation(currInst);
                    breakpoint.setInstID(currInst.getInstID());
                    breakpoint.setBPInstructionStatus(status);
                    breakpoint.setBPInstructionLocation(location);
                    instructions.set(i, breakpoint);
                    DMLBreakpointManager.updateBreakpoint(lineNumber, status);
                } else {
                    instructions.remove(i);
                    DMLBreakpointManager.removeBreakpoint(lineNumber, status);
                }
                return;
            }
        }
    }
}