Examples with LopsException org.apache.sysml.lops.LopsException used on opensource projects

Example 1 with LopsException

use of org.apache.sysml.lops.LopsException in project incubator-systemml by apache.

the class DMLProgram method createRuntimeProgramBlock.

public ProgramBlock createRuntimeProgramBlock(Program prog, StatementBlock sb, DMLConfig config) throws IOException, LopsException, DMLRuntimeException {
    Dag<Lop> dag = null;
    Dag<Lop> pred_dag = null;
    ArrayList<Instruction> instruct;
    ArrayList<Instruction> pred_instruct = null;
    ProgramBlock retPB = null;
    // process While Statement - add runtime program blocks to program
    if (sb instanceof WhileStatementBlock) {
        // create DAG for loop predicates
        pred_dag = new Dag<Lop>();
        ((WhileStatementBlock) sb).get_predicateLops().addToDag(pred_dag);
        // create instructions for loop predicates
        pred_instruct = new ArrayList<Instruction>();
        ArrayList<Instruction> pInst = pred_dag.getJobs(null, config);
        for (Instruction i : pInst) {
            pred_instruct.add(i);
        }
        // create while program block
        WhileProgramBlock rtpb = new WhileProgramBlock(prog, pred_instruct);
        if (rtpb.getPredicateResultVar() == null) {
            // e.g case : WHILE(continue)
            if (((WhileStatementBlock) sb).get_predicateLops().getExecLocation() == LopProperties.ExecLocation.Data) {
                String resultVar = ((WhileStatementBlock) sb).get_predicateLops().getOutputParameters().getLabel();
                rtpb.setPredicateResultVar(resultVar);
            } else {
                LOG.error(sb.printBlockErrorLocation() + "Error in translating the WHILE predicate.");
                throw new LopsException(sb.printBlockErrorLocation() + "Error in translating the WHILE predicate.");
            }
        }
        //// process the body of the while statement block ////
        WhileStatementBlock wsb = (WhileStatementBlock) sb;
        if (wsb.getNumStatements() > 1) {
            LOG.error(wsb.printBlockErrorLocation() + "WhileStatementBlock should only have 1 statement");
            throw new LopsException(wsb.printBlockErrorLocation() + "WhileStatementBlock should only have 1 statement");
        }
        WhileStatement wstmt = (WhileStatement) wsb.getStatement(0);
        for (StatementBlock sblock : wstmt.getBody()) {
            // process the body
            ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
            rtpb.addProgramBlock(childBlock);
        }
        // check there are actually Lops in to process (loop stmt body will not have any)
        if (wsb.getLops() != null && !wsb.getLops().isEmpty()) {
            LOG.error(wsb.printBlockErrorLocation() + "WhileStatementBlock should have no Lops");
            throw new LopsException(wsb.printBlockErrorLocation() + "WhileStatementBlock should have no Lops");
        }
        retPB = rtpb;
        //post processing for generating missing instructions
        //retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
        // add statement block
        retPB.setStatementBlock(sb);
        // add location information
        retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
    } else // process If Statement - add runtime program blocks to program
    if (sb instanceof IfStatementBlock) {
        // create DAG for loop predicates
        pred_dag = new Dag<Lop>();
        ((IfStatementBlock) sb).get_predicateLops().addToDag(pred_dag);
        // create instructions for loop predicates
        pred_instruct = new ArrayList<Instruction>();
        ArrayList<Instruction> pInst = pred_dag.getJobs(null, config);
        for (Instruction i : pInst) {
            pred_instruct.add(i);
        }
        // create if program block
        IfProgramBlock rtpb = new IfProgramBlock(prog, pred_instruct);
        if (rtpb.getPredicateResultVar() == null) {
            // e.g case : If(continue)
            if (((IfStatementBlock) sb).get_predicateLops().getExecLocation() == LopProperties.ExecLocation.Data) {
                String resultVar = ((IfStatementBlock) sb).get_predicateLops().getOutputParameters().getLabel();
                rtpb.setPredicateResultVar(resultVar);
            } else {
                LOG.error(sb.printBlockErrorLocation() + "Error in translating the IF predicate.");
                throw new LopsException(sb.printBlockErrorLocation() + "Error in translating the IF predicate.");
            }
        }
        // process the body of the if statement block
        IfStatementBlock isb = (IfStatementBlock) sb;
        if (isb.getNumStatements() > 1) {
            LOG.error(isb.printBlockErrorLocation() + "IfStatementBlock should have only 1 statement");
            throw new LopsException(isb.printBlockErrorLocation() + "IfStatementBlock should have only 1 statement");
        }
        IfStatement istmt = (IfStatement) isb.getStatement(0);
        // process the if body
        for (StatementBlock sblock : istmt.getIfBody()) {
            ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
            rtpb.addProgramBlockIfBody(childBlock);
        }
        // process the else body
        for (StatementBlock sblock : istmt.getElseBody()) {
            ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
            rtpb.addProgramBlockElseBody(childBlock);
        }
        // check there are actually Lops in to process (loop stmt body will not have any)
        if (isb.getLops() != null && !isb.getLops().isEmpty()) {
            LOG.error(isb.printBlockErrorLocation() + "IfStatementBlock should have no Lops");
            throw new LopsException(isb.printBlockErrorLocation() + "IfStatementBlock should have no Lops");
        }
        retPB = rtpb;
        //post processing for generating missing instructions
        //retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
        // add statement block
        retPB.setStatementBlock(sb);
        // add location information
        retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
    } else // NOTE: applies to ForStatementBlock and ParForStatementBlock
    if (sb instanceof ForStatementBlock) {
        ForStatementBlock fsb = (ForStatementBlock) sb;
        // create DAGs for loop predicates 
        Dag<Lop> fromDag = new Dag<Lop>();
        Dag<Lop> toDag = new Dag<Lop>();
        Dag<Lop> incrementDag = new Dag<Lop>();
        if (fsb.getFromHops() != null)
            fsb.getFromLops().addToDag(fromDag);
        if (fsb.getToHops() != null)
            fsb.getToLops().addToDag(toDag);
        if (fsb.getIncrementHops() != null)
            fsb.getIncrementLops().addToDag(incrementDag);
        // create instructions for loop predicates			
        ArrayList<Instruction> fromInstructions = fromDag.getJobs(null, config);
        ArrayList<Instruction> toInstructions = toDag.getJobs(null, config);
        ArrayList<Instruction> incrementInstructions = incrementDag.getJobs(null, config);
        // create for program block
        String sbName = null;
        ForProgramBlock rtpb = null;
        IterablePredicate iterPred = fsb.getIterPredicate();
        String[] iterPredData = IterablePredicate.createIterablePredicateVariables(iterPred.getIterVar().getName(), fsb.getFromLops(), fsb.getToLops(), fsb.getIncrementLops());
        if (sb instanceof ParForStatementBlock) {
            sbName = "ParForStatementBlock";
            rtpb = new ParForProgramBlock(prog, iterPredData, iterPred.getParForParams());
            ParForProgramBlock pfrtpb = (ParForProgramBlock) rtpb;
            pfrtpb.setResultVariables(((ParForStatementBlock) sb).getResultVariables());
            //used for optimization and creating unscoped variables
            pfrtpb.setStatementBlock((ParForStatementBlock) sb);
        } else //ForStatementBlock
        {
            sbName = "ForStatementBlock";
            rtpb = new ForProgramBlock(prog, iterPredData);
        }
        rtpb.setFromInstructions(fromInstructions);
        rtpb.setToInstructions(toInstructions);
        rtpb.setIncrementInstructions(incrementInstructions);
        rtpb.setIterablePredicateVars(iterPredData);
        // process the body of the for statement block
        if (fsb.getNumStatements() > 1) {
            LOG.error(fsb.printBlockErrorLocation() + " " + sbName + " should have 1 statement");
            throw new LopsException(fsb.printBlockErrorLocation() + " " + sbName + " should have 1 statement");
        }
        ForStatement fs = (ForStatement) fsb.getStatement(0);
        for (StatementBlock sblock : fs.getBody()) {
            ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
            rtpb.addProgramBlock(childBlock);
        }
        // check there are actually Lops in to process (loop stmt body will not have any)
        if (fsb.getLops() != null && !fsb.getLops().isEmpty()) {
            LOG.error(fsb.printBlockErrorLocation() + sbName + " should have no Lops");
            throw new LopsException(fsb.printBlockErrorLocation() + sbName + " should have no Lops");
        }
        retPB = rtpb;
        //post processing for generating missing instructions
        //retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
        // add statement block
        retPB.setStatementBlock(sb);
        // add location information
        retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
    } else // process function statement block - add runtime program blocks to program
    if (sb instanceof FunctionStatementBlock) {
        FunctionStatementBlock fsb = (FunctionStatementBlock) sb;
        if (fsb.getNumStatements() > 1) {
            LOG.error(fsb.printBlockErrorLocation() + "FunctionStatementBlock should only have 1 statement");
            throw new LopsException(fsb.printBlockErrorLocation() + "FunctionStatementBlock should only have 1 statement");
        }
        FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0);
        FunctionProgramBlock rtpb = null;
        if (fstmt instanceof ExternalFunctionStatement) {
            // create external function program block
            String execType = ((ExternalFunctionStatement) fstmt).getOtherParams().get(ExternalFunctionStatement.EXEC_TYPE);
            boolean isCP = (execType.equals(ExternalFunctionStatement.IN_MEMORY)) ? true : false;
            String scratchSpaceLoc = null;
            try {
                scratchSpaceLoc = config.getTextValue(DMLConfig.SCRATCH_SPACE);
            } catch (Exception e) {
                LOG.error(fsb.printBlockErrorLocation() + "could not retrieve parameter " + DMLConfig.SCRATCH_SPACE + " from DMLConfig");
            }
            StringBuilder buff = new StringBuilder();
            buff.append(scratchSpaceLoc);
            buff.append(Lop.FILE_SEPARATOR);
            buff.append(Lop.PROCESS_PREFIX);
            buff.append(DMLScript.getUUID());
            buff.append(Lop.FILE_SEPARATOR);
            buff.append(ProgramConverter.CP_ROOT_THREAD_ID);
            buff.append(Lop.FILE_SEPARATOR);
            buff.append("PackageSupport");
            buff.append(Lop.FILE_SEPARATOR);
            String basedir = buff.toString();
            if (isCP) {
                rtpb = new ExternalFunctionProgramBlockCP(prog, fstmt.getInputParams(), fstmt.getOutputParams(), ((ExternalFunctionStatement) fstmt).getOtherParams(), basedir);
            } else {
                rtpb = new ExternalFunctionProgramBlock(prog, fstmt.getInputParams(), fstmt.getOutputParams(), ((ExternalFunctionStatement) fstmt).getOtherParams(), basedir);
            }
            if (!fstmt.getBody().isEmpty()) {
                LOG.error(fstmt.printErrorLocation() + "ExternalFunctionStatementBlock should have no statement blocks in body");
                throw new LopsException(fstmt.printErrorLocation() + "ExternalFunctionStatementBlock should have no statement blocks in body");
            }
        } else {
            // create function program block
            rtpb = new FunctionProgramBlock(prog, fstmt.getInputParams(), fstmt.getOutputParams());
            // process the function statement body
            for (StatementBlock sblock : fstmt.getBody()) {
                // process the body
                ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
                rtpb.addProgramBlock(childBlock);
            }
        }
        // check there are actually Lops in to process (loop stmt body will not have any)
        if (fsb.getLops() != null && !fsb.getLops().isEmpty()) {
            LOG.error(fsb.printBlockErrorLocation() + "FunctionStatementBlock should have no Lops");
            throw new LopsException(fsb.printBlockErrorLocation() + "FunctionStatementBlock should have no Lops");
        }
        retPB = rtpb;
        // add location information
        retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
    } else {
        // handle general case
        ProgramBlock rtpb = new ProgramBlock(prog);
        // DAGs for Lops
        dag = new Dag<Lop>();
        // check there are actually Lops in to process (loop stmt body will not have any)
        if (sb.getLops() != null && !sb.getLops().isEmpty()) {
            for (Lop l : sb.getLops()) {
                l.addToDag(dag);
            }
            // Instructions for Lobs DAGs
            instruct = dag.getJobs(sb, config);
            rtpb.addInstructions(instruct);
        }
        /*// TODO: check with Doug
			// add instruction for a function call
			if (sb.getFunctionCallInst() != null){
				rtpb.addInstruction(sb.getFunctionCallInst());
			}*/
        retPB = rtpb;
        //post processing for generating missing instructions
        //retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
        // add statement block
        retPB.setStatementBlock(sb);
        // add location information
        retPB.setAllPositions(sb.getBeginLine(), sb.getBeginColumn(), sb.getEndLine(), sb.getEndColumn());
    }
    return retPB;
}

Example 2 with LopsException

use of org.apache.sysml.lops.LopsException in project incubator-systemml by apache.

the class LiteralOp method constructLops.

@Override
public Lop constructLops() {
    // return already created lops
    if (getLops() != null)
        return getLops();
    try {
        Lop l = null;
        switch(getValueType()) {
            case DOUBLE:
                l = Data.createLiteralLop(ValueType.DOUBLE, Double.toString(value_double));
                break;
            case BOOLEAN:
                l = Data.createLiteralLop(ValueType.BOOLEAN, Boolean.toString(value_boolean));
                break;
            case STRING:
                l = Data.createLiteralLop(ValueType.STRING, value_string);
                break;
            case INT:
                l = Data.createLiteralLop(ValueType.INT, Long.toString(value_long));
                break;
            default:
                throw new HopsException(this.printErrorLocation() + "unexpected value type constructing lops for LiteralOp.\n");
        }
        l.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
        setLineNumbers(l);
        setLops(l);
    } catch (LopsException e) {
        throw new HopsException(e);
    }
    return getLops();
}

Example 3 with LopsException

use of org.apache.sysml.lops.LopsException in project incubator-systemml by apache.

the class DataOp method constructLops.

@Override
public Lop constructLops() {
    // return already created lops
    if (getLops() != null)
        return getLops();
    ExecType et = optFindExecType();
    Lop l = null;
    // construct lops for all input parameters
    HashMap<String, Lop> inputLops = new HashMap<>();
    for (Entry<String, Integer> cur : _paramIndexMap.entrySet()) {
        inputLops.put(cur.getKey(), getInput().get(cur.getValue()).constructLops());
    }
    // Create the lop
    switch(_dataop) {
        case TRANSIENTREAD:
            l = new Data(HopsData2Lops.get(_dataop), null, inputLops, getName(), null, getDataType(), getValueType(), true, getInputFormatType());
            setOutputDimensions(l);
            break;
        case PERSISTENTREAD:
            l = new Data(HopsData2Lops.get(_dataop), null, inputLops, getName(), null, getDataType(), getValueType(), false, getInputFormatType());
            l.getOutputParameters().setDimensions(getDim1(), getDim2(), _inRowsInBlock, _inColsInBlock, getNnz(), getUpdateType());
            break;
        case PERSISTENTWRITE:
            l = new Data(HopsData2Lops.get(_dataop), getInput().get(0).constructLops(), inputLops, getName(), null, getDataType(), getValueType(), false, getInputFormatType());
            ((Data) l).setExecType(et);
            setOutputDimensions(l);
            break;
        case TRANSIENTWRITE:
            l = new Data(HopsData2Lops.get(_dataop), getInput().get(0).constructLops(), inputLops, getName(), null, getDataType(), getValueType(), true, getInputFormatType());
            setOutputDimensions(l);
            break;
        case FUNCTIONOUTPUT:
            l = new Data(HopsData2Lops.get(_dataop), getInput().get(0).constructLops(), inputLops, getName(), null, getDataType(), getValueType(), true, getInputFormatType());
            ((Data) l).setExecType(et);
            setOutputDimensions(l);
            break;
        default:
            throw new LopsException("Invalid operation type for Data LOP: " + _dataop);
    }
    setLineNumbers(l);
    setLops(l);
    // add reblock/checkpoint lops if necessary
    constructAndSetLopsDataFlowProperties();
    return getLops();
}

Example 4 with LopsException

use of org.apache.sysml.lops.LopsException in project incubator-systemml by apache.

the class Hop method constructAndSetCompressionLopIfRequired.

private void constructAndSetCompressionLopIfRequired() {
    // determine execution type
    ExecType et = ExecType.CP;
    if (OptimizerUtils.isSparkExecutionMode() && getDataType() != DataType.SCALAR) {
        // persist and unpersist calls (4x the memory budget is conservative)
        if (OptimizerUtils.isHybridExecutionMode() && 2 * _outputMemEstimate < OptimizerUtils.getLocalMemBudget() || _etypeForced == ExecType.CP) {
            et = ExecType.CP;
        } else // default case
        {
            et = ExecType.SPARK;
        }
    }
    // add reblock lop to output if required
    if (_requiresCompression) {
        try {
            Lop compress = new Compression(getLops(), getDataType(), getValueType(), et);
            setOutputDimensions(compress);
            setLineNumbers(compress);
            setLops(compress);
        } catch (LopsException ex) {
            throw new HopsException(ex);
        }
    }
}

Example 5 with LopsException

use of org.apache.sysml.lops.LopsException in project incubator-systemml by apache.

the class Dag method generateControlProgramJobs.

/**
 * Method to generate instructions that are executed in Control Program. At
 * this point, this DAG has no dependencies on the MR dag. ie. none of the
 * inputs are outputs of MR jobs
 *
 * @param execNodes list of low-level operators
 * @param inst list of instructions
 * @param writeInst list of write instructions
 * @param deleteInst list of delete instructions
 */
private void generateControlProgramJobs(ArrayList<Lop> execNodes, ArrayList<Instruction> inst, ArrayList<Instruction> writeInst, ArrayList<Instruction> deleteInst) {
    // nodes to be deleted from execnodes
    ArrayList<Lop> markedNodes = new ArrayList<>();
    // variable names to be deleted
    ArrayList<String> var_deletions = new ArrayList<>();
    HashMap<String, Lop> var_deletionsLineNum = new HashMap<>();
    boolean doRmVar = false;
    for (int i = 0; i < execNodes.size(); i++) {
        Lop node = execNodes.get(i);
        doRmVar = false;
        // TODO: statiko -- check if this condition ever evaluated to TRUE
        if (node.getExecLocation() == ExecLocation.Data && ((Data) node).getOperationType() == Data.OperationTypes.READ && ((Data) node).getDataType() == DataType.SCALAR && node.getOutputParameters().getFile_name() == null) {
            markedNodes.add(node);
            continue;
        }
        // output scalar instructions and mark nodes for deletion
        if (node.getExecLocation() == ExecLocation.ControlProgram) {
            if (node.getDataType() == DataType.SCALAR) {
                // Output from lops with SCALAR data type must
                // go into Temporary Variables (Var0, Var1, etc.)
                NodeOutput out = setupNodeOutputs(node, ExecType.CP, false, false);
                // dummy
                inst.addAll(out.getPreInstructions());
                deleteInst.addAll(out.getLastInstructions());
            } else {
                // Output from lops with non-SCALAR data type must
                // go into Temporary Files (temp0, temp1, etc.)
                NodeOutput out = setupNodeOutputs(node, ExecType.CP, false, false);
                inst.addAll(out.getPreInstructions());
                boolean hasTransientWriteParent = false;
                for (Lop parent : node.getOutputs()) {
                    if (parent.getExecLocation() == ExecLocation.Data && ((Data) parent).getOperationType() == Data.OperationTypes.WRITE && ((Data) parent).isTransient()) {
                        hasTransientWriteParent = true;
                        break;
                    }
                }
                if (!hasTransientWriteParent) {
                    deleteInst.addAll(out.getLastInstructions());
                } else {
                    var_deletions.add(node.getOutputParameters().getLabel());
                    var_deletionsLineNum.put(node.getOutputParameters().getLabel(), node);
                }
            }
            String inst_string = "";
            // are handled separately, by simply passing ONLY the output variable to getInstructions()
            if (node.getType() == Lop.Type.ParameterizedBuiltin || node.getType() == Lop.Type.GroupedAgg || node.getType() == Lop.Type.DataGen) {
                inst_string = node.getInstructions(node.getOutputParameters().getLabel());
            } else // separately as well by passing arrays of inputs and outputs
            if (node.getType() == Lop.Type.FunctionCallCP) {
                String[] inputs = new String[node.getInputs().size()];
                String[] outputs = new String[node.getOutputs().size()];
                int count = 0;
                for (Lop in : node.getInputs()) inputs[count++] = in.getOutputParameters().getLabel();
                count = 0;
                for (Lop out : node.getOutputs()) outputs[count++] = out.getOutputParameters().getLabel();
                inst_string = node.getInstructions(inputs, outputs);
            } else if (node.getType() == Lop.Type.Nary) {
                String[] inputs = new String[node.getInputs().size()];
                int count = 0;
                for (Lop in : node.getInputs()) inputs[count++] = in.getOutputParameters().getLabel();
                inst_string = node.getInstructions(inputs, node.getOutputParameters().getLabel());
            } else {
                if (node.getInputs().isEmpty()) {
                    // currently, such a case exists only for Rand lop
                    inst_string = node.getInstructions(node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 1) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 2) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 3 || node.getType() == Type.Ctable) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 4) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getInputs().get(3).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 5) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getInputs().get(3).getOutputParameters().getLabel(), node.getInputs().get(4).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 6) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getInputs().get(3).getOutputParameters().getLabel(), node.getInputs().get(4).getOutputParameters().getLabel(), node.getInputs().get(5).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else if (node.getInputs().size() == 7) {
                    inst_string = node.getInstructions(node.getInputs().get(0).getOutputParameters().getLabel(), node.getInputs().get(1).getOutputParameters().getLabel(), node.getInputs().get(2).getOutputParameters().getLabel(), node.getInputs().get(3).getOutputParameters().getLabel(), node.getInputs().get(4).getOutputParameters().getLabel(), node.getInputs().get(5).getOutputParameters().getLabel(), node.getInputs().get(6).getOutputParameters().getLabel(), node.getOutputParameters().getLabel());
                } else {
                    String[] inputs = new String[node.getInputs().size()];
                    for (int j = 0; j < node.getInputs().size(); j++) inputs[j] = node.getInputs().get(j).getOutputParameters().getLabel();
                    inst_string = node.getInstructions(inputs, node.getOutputParameters().getLabel());
                }
            }
            try {
                if (LOG.isTraceEnabled())
                    LOG.trace("Generating instruction - " + inst_string);
                Instruction currInstr = InstructionParser.parseSingleInstruction(inst_string);
                if (currInstr == null) {
                    throw new LopsException("Error parsing the instruction:" + inst_string);
                }
                if (node._beginLine != 0)
                    currInstr.setLocation(node);
                else if (!node.getOutputs().isEmpty())
                    currInstr.setLocation(node.getOutputs().get(0));
                else if (!node.getInputs().isEmpty())
                    currInstr.setLocation(node.getInputs().get(0));
                inst.add(currInstr);
            } catch (Exception e) {
                throw new LopsException(node.printErrorLocation() + "Problem generating simple inst - " + inst_string, e);
            }
            markedNodes.add(node);
            doRmVar = true;
        } else if (node.getExecLocation() == ExecLocation.Data) {
            Data dnode = (Data) node;
            Data.OperationTypes op = dnode.getOperationType();
            if (op == Data.OperationTypes.WRITE) {
                NodeOutput out = null;
                if (sendWriteLopToMR(node)) {
                    // In this case, Data WRITE lop goes into MR, and
                    // we don't have to do anything here
                    doRmVar = false;
                } else {
                    out = setupNodeOutputs(node, ExecType.CP, false, false);
                    if (dnode.getDataType() == DataType.SCALAR) {
                        // processing is same for both transient and persistent scalar writes
                        writeInst.addAll(out.getLastInstructions());
                        doRmVar = false;
                    } else {
                        // setupNodeOutputs() handles both transient and persistent matrix writes
                        if (dnode.isTransient()) {
                            deleteInst.addAll(out.getLastInstructions());
                            doRmVar = false;
                        } else {
                            // In case of persistent write lop, write instruction will be generated
                            // and that instruction must be added to <code>inst</code> so that it gets
                            // executed immediately. If it is added to <code>deleteInst</code> then it
                            // gets executed at the end of program block's execution
                            inst.addAll(out.getLastInstructions());
                            doRmVar = true;
                        }
                    }
                    markedNodes.add(node);
                }
            } else {
                // generate a temp label to hold the value that is read from HDFS
                if (node.getDataType() == DataType.SCALAR) {
                    node.getOutputParameters().setLabel(Lop.SCALAR_VAR_NAME_PREFIX + var_index.getNextID());
                    String io_inst = node.getInstructions(node.getOutputParameters().getLabel(), node.getOutputParameters().getFile_name());
                    CPInstruction currInstr = CPInstructionParser.parseSingleInstruction(io_inst);
                    currInstr.setLocation(node);
                    inst.add(currInstr);
                    Instruction tempInstr = VariableCPInstruction.prepareRemoveInstruction(node.getOutputParameters().getLabel());
                    tempInstr.setLocation(node);
                    deleteInst.add(tempInstr);
                } else {
                    throw new LopsException("Matrix READs are not handled in CP yet!");
                }
                markedNodes.add(node);
                doRmVar = true;
            }
        }
        // see if rmvar instructions can be generated for node's inputs
        if (doRmVar)
            processConsumersForInputs(node, inst, deleteInst);
        doRmVar = false;
    }
    for (String var : var_deletions) {
        Instruction rmInst = VariableCPInstruction.prepareRemoveInstruction(var);
        if (LOG.isTraceEnabled())
            LOG.trace("  Adding var_deletions: " + rmInst.toString());
        rmInst.setLocation(var_deletionsLineNum.get(var));
        deleteInst.add(rmInst);
    }
    // delete all marked nodes
    for (Lop node : markedNodes) {
        execNodes.remove(node);
    }
}