Examples with Lop org.apache.sysml.lops.Lop used on opensource projects

Example 56 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class Dag method getMapperInstructions.

/**
 * Method to get mapper instructions for a MR job.
 *
 * @param node low-level operator
 * @param execNodes list of exec nodes
 * @param inputStrings list of input strings
 * @param instructionsInMapper list of instructions in mapper
 * @param nodeIndexMapping ?
 * @param start_index starting index
 * @param inputLabels input labels
 * @param MRJoblineNumbers MR job line numbers
 * @return -1 if problem
 */
private int getMapperInstructions(Lop node, ArrayList<Lop> execNodes, ArrayList<String> inputStrings, ArrayList<String> instructionsInMapper, HashMap<Lop, Integer> nodeIndexMapping, int[] start_index, ArrayList<String> inputLabels, ArrayList<Lop> inputLops, ArrayList<Integer> MRJobLineNumbers) {
    // if input source, return index
    if (nodeIndexMapping.containsKey(node))
        return nodeIndexMapping.get(node);
    // not input source and not in exec nodes, then return.
    if (!execNodes.contains(node))
        return -1;
    ArrayList<Integer> inputIndices = new ArrayList<>();
    int max_input_index = -1;
    // get mapper instructions
    for (Lop childNode : node.getInputs()) {
        int ret_val = getMapperInstructions(childNode, execNodes, inputStrings, instructionsInMapper, nodeIndexMapping, start_index, inputLabels, inputLops, MRJobLineNumbers);
        inputIndices.add(ret_val);
        if (ret_val > max_input_index) {
            max_input_index = ret_val;
        }
    }
    // to mapper instructions.
    if ((node.getExecLocation() == ExecLocation.Map || node.getExecLocation() == ExecLocation.MapOrReduce) && !hasChildNode(node, execNodes, ExecLocation.MapAndReduce) && !hasChildNode(node, execNodes, ExecLocation.Reduce)) {
        int output_index = max_input_index;
        // cannot reuse index if this is true
        // need to add better indexing schemes
        output_index = start_index[0];
        start_index[0]++;
        nodeIndexMapping.put(node, output_index);
        if (node instanceof Unary && node.getInputs().size() > 1) {
            // Following code must be executed only for those Unary
            // operators that have more than one input
            // It should not be executed for "true" unary operators like
            // cos(A).
            int index = 0;
            for (int i1 = 0; i1 < node.getInputs().size(); i1++) {
                if (node.getInputs().get(i1).getDataType() == DataType.SCALAR) {
                    index = i1;
                    break;
                }
            }
            // if data lop not a literal -- add label
            if (node.getInputs().get(index).getExecLocation() == ExecLocation.Data && !((Data) (node.getInputs().get(index))).isLiteral()) {
                inputLabels.add(node.getInputs().get(index).getOutputParameters().getLabel());
                inputLops.add(node.getInputs().get(index));
            }
            // if not data lop, then this is an intermediate variable.
            if (node.getInputs().get(index).getExecLocation() != ExecLocation.Data) {
                inputLabels.add(node.getInputs().get(index).getOutputParameters().getLabel());
                inputLops.add(node.getInputs().get(index));
            }
        }
        // get mapper instruction.
        if (node.getInputs().size() == 1)
            instructionsInMapper.add(node.getInstructions(inputIndices.get(0), output_index));
        else if (node.getInputs().size() == 2) {
            instructionsInMapper.add(node.getInstructions(inputIndices.get(0), inputIndices.get(1), output_index));
        } else if (node.getInputs().size() == 3)
            instructionsInMapper.add(node.getInstructions(inputIndices.get(0), inputIndices.get(1), inputIndices.get(2), output_index));
        else if (node.getInputs().size() == 4) {
            // Example: Reshape
            instructionsInMapper.add(node.getInstructions(inputIndices.get(0), inputIndices.get(1), inputIndices.get(2), inputIndices.get(3), output_index));
        } else if (node.getInputs().size() == 5) {
            // Example: RangeBasedReIndex A[row_l:row_u, col_l:col_u]
            instructionsInMapper.add(node.getInstructions(inputIndices.get(0), inputIndices.get(1), inputIndices.get(2), inputIndices.get(3), inputIndices.get(4), output_index));
        } else if (node.getInputs().size() == 7) {
            // Example: RangeBasedReIndex A[row_l:row_u, col_l:col_u] = B
            instructionsInMapper.add(node.getInstructions(inputIndices.get(0), inputIndices.get(1), inputIndices.get(2), inputIndices.get(3), inputIndices.get(4), inputIndices.get(5), inputIndices.get(6), output_index));
        } else
            throw new LopsException("Node with " + node.getInputs().size() + " inputs is not supported in dag.java.");
        if (DMLScript.ENABLE_DEBUG_MODE) {
            MRJobLineNumbers.add(node._beginLine);
        }
        return output_index;
    }
    return -1;
}

Example 57 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class DMLTranslator method createRuntimeProgramBlock.

public ProgramBlock createRuntimeProgramBlock(Program prog, StatementBlock sb, DMLConfig config) {
    Dag<Lop> dag = null;
    Dag<Lop> pred_dag = null;
    ArrayList<Instruction> instruct;
    ArrayList<Instruction> pred_instruct = null;
    ProgramBlock retPB = null;
    // process While Statement - add runtime program blocks to program
    if (sb instanceof WhileStatementBlock) {
        // create DAG for loop predicates
        pred_dag = new Dag<>();
        ((WhileStatementBlock) sb).get_predicateLops().addToDag(pred_dag);
        // create instructions for loop predicates
        pred_instruct = new ArrayList<>();
        ArrayList<Instruction> pInst = pred_dag.getJobs(null, config);
        for (Instruction i : pInst) {
            pred_instruct.add(i);
        }
        // create while program block
        WhileProgramBlock rtpb = new WhileProgramBlock(prog, pred_instruct);
        // // process the body of the while statement block ////
        WhileStatementBlock wsb = (WhileStatementBlock) sb;
        WhileStatement wstmt = (WhileStatement) wsb.getStatement(0);
        for (StatementBlock sblock : wstmt.getBody()) {
            // process the body
            ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
            rtpb.addProgramBlock(childBlock);
        }
        retPB = rtpb;
        // add statement block
        retPB.setStatementBlock(sb);
        // add location information
        retPB.setParseInfo(sb);
    } else // process If Statement - add runtime program blocks to program
    if (sb instanceof IfStatementBlock) {
        // create DAG for loop predicates
        pred_dag = new Dag<>();
        ((IfStatementBlock) sb).get_predicateLops().addToDag(pred_dag);
        // create instructions for loop predicates
        pred_instruct = new ArrayList<>();
        ArrayList<Instruction> pInst = pred_dag.getJobs(null, config);
        for (Instruction i : pInst) {
            pred_instruct.add(i);
        }
        // create if program block
        IfProgramBlock rtpb = new IfProgramBlock(prog, pred_instruct);
        // process the body of the if statement block
        IfStatementBlock isb = (IfStatementBlock) sb;
        IfStatement istmt = (IfStatement) isb.getStatement(0);
        // process the if body
        for (StatementBlock sblock : istmt.getIfBody()) {
            ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
            rtpb.addProgramBlockIfBody(childBlock);
        }
        // process the else body
        for (StatementBlock sblock : istmt.getElseBody()) {
            ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
            rtpb.addProgramBlockElseBody(childBlock);
        }
        retPB = rtpb;
        // post processing for generating missing instructions
        // retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
        // add statement block
        retPB.setStatementBlock(sb);
        // add location information
        retPB.setParseInfo(sb);
    } else // NOTE: applies to ForStatementBlock and ParForStatementBlock
    if (sb instanceof ForStatementBlock) {
        ForStatementBlock fsb = (ForStatementBlock) sb;
        // create DAGs for loop predicates
        Dag<Lop> fromDag = new Dag<>();
        Dag<Lop> toDag = new Dag<>();
        Dag<Lop> incrementDag = new Dag<>();
        if (fsb.getFromHops() != null)
            fsb.getFromLops().addToDag(fromDag);
        if (fsb.getToHops() != null)
            fsb.getToLops().addToDag(toDag);
        if (fsb.getIncrementHops() != null)
            fsb.getIncrementLops().addToDag(incrementDag);
        // create instructions for loop predicates
        ArrayList<Instruction> fromInstructions = fromDag.getJobs(null, config);
        ArrayList<Instruction> toInstructions = toDag.getJobs(null, config);
        ArrayList<Instruction> incrementInstructions = incrementDag.getJobs(null, config);
        // create for program block
        ForProgramBlock rtpb = null;
        IterablePredicate iterPred = fsb.getIterPredicate();
        if (sb instanceof ParForStatementBlock && ConfigurationManager.isParallelParFor()) {
            rtpb = new ParForProgramBlock(prog, iterPred.getIterVar().getName(), iterPred.getParForParams(), ((ParForStatementBlock) sb).getResultVariables());
            ParForProgramBlock pfrtpb = (ParForProgramBlock) rtpb;
            // used for optimization and creating unscoped variables
            pfrtpb.setStatementBlock((ParForStatementBlock) sb);
        } else {
            // ForStatementBlock
            rtpb = new ForProgramBlock(prog, iterPred.getIterVar().getName());
        }
        rtpb.setFromInstructions(fromInstructions);
        rtpb.setToInstructions(toInstructions);
        rtpb.setIncrementInstructions(incrementInstructions);
        // process the body of the for statement block
        ForStatement fs = (ForStatement) fsb.getStatement(0);
        for (StatementBlock sblock : fs.getBody()) {
            ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
            rtpb.addProgramBlock(childBlock);
        }
        retPB = rtpb;
        // add statement block
        retPB.setStatementBlock(sb);
        // add location information
        retPB.setParseInfo(sb);
    } else // process function statement block - add runtime program blocks to program
    if (sb instanceof FunctionStatementBlock) {
        FunctionStatementBlock fsb = (FunctionStatementBlock) sb;
        FunctionStatement fstmt = (FunctionStatement) fsb.getStatement(0);
        FunctionProgramBlock rtpb = null;
        if (fstmt instanceof ExternalFunctionStatement) {
            // create external function program block
            String execType = ((ExternalFunctionStatement) fstmt).getOtherParams().get(ExternalFunctionStatement.EXEC_TYPE);
            boolean isCP = (execType.equals(ExternalFunctionStatement.IN_MEMORY)) ? true : false;
            StringBuilder buff = new StringBuilder();
            buff.append(config.getTextValue(DMLConfig.SCRATCH_SPACE));
            buff.append(Lop.FILE_SEPARATOR);
            buff.append(Lop.PROCESS_PREFIX);
            buff.append(DMLScript.getUUID());
            buff.append(Lop.FILE_SEPARATOR);
            buff.append(ProgramConverter.CP_ROOT_THREAD_ID);
            buff.append(Lop.FILE_SEPARATOR);
            buff.append("PackageSupport");
            buff.append(Lop.FILE_SEPARATOR);
            String basedir = buff.toString();
            if (isCP) {
                rtpb = new ExternalFunctionProgramBlockCP(prog, fstmt.getInputParams(), fstmt.getOutputParams(), ((ExternalFunctionStatement) fstmt).getOtherParams(), basedir);
            } else {
                rtpb = new ExternalFunctionProgramBlock(prog, fstmt.getInputParams(), fstmt.getOutputParams(), ((ExternalFunctionStatement) fstmt).getOtherParams(), basedir);
            }
            if (!fstmt.getBody().isEmpty()) {
                LOG.error(fstmt.printErrorLocation() + "ExternalFunctionStatementBlock should have no statement blocks in body");
                throw new LopsException(fstmt.printErrorLocation() + "ExternalFunctionStatementBlock should have no statement blocks in body");
            }
        } else {
            // create function program block
            rtpb = new FunctionProgramBlock(prog, fstmt.getInputParams(), fstmt.getOutputParams());
            // process the function statement body
            for (StatementBlock sblock : fstmt.getBody()) {
                // process the body
                ProgramBlock childBlock = createRuntimeProgramBlock(prog, sblock, config);
                rtpb.addProgramBlock(childBlock);
            }
        }
        // check there are actually Lops in to process (loop stmt body will not have any)
        if (fsb.getLops() != null && !fsb.getLops().isEmpty()) {
            LOG.error(fsb.printBlockErrorLocation() + "FunctionStatementBlock should have no Lops");
            throw new LopsException(fsb.printBlockErrorLocation() + "FunctionStatementBlock should have no Lops");
        }
        retPB = rtpb;
        // add location information
        retPB.setParseInfo(sb);
    } else {
        // handle general case
        ProgramBlock rtpb = new ProgramBlock(prog);
        // DAGs for Lops
        dag = new Dag<>();
        // check there are actually Lops in to process (loop stmt body will not have any)
        if (sb.getLops() != null && !sb.getLops().isEmpty()) {
            for (Lop l : sb.getLops()) {
                l.addToDag(dag);
            }
            // Instructions for Lops DAGs
            instruct = dag.getJobs(sb, config);
            rtpb.addInstructions(instruct);
        }
        retPB = rtpb;
        // post processing for generating missing instructions
        // retPB = verifyAndCorrectProgramBlock(sb.liveIn(), sb.liveOut(), sb._kill, retPB);
        // add statement block
        retPB.setStatementBlock(sb);
        // add location information
        retPB.setParseInfo(sb);
    }
    return retPB;
}

Example 58 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class AggBinaryOp method constructSparkLopsMapMM.

private void constructSparkLopsMapMM(MMultMethod method) {
    Lop mapmult = null;
    if (isLeftTransposeRewriteApplicable(false, false)) {
        mapmult = constructSparkLopsMapMMWithLeftTransposeRewrite();
    } else {
        // If number of columns is smaller than block size then explicit aggregation is not required.
        // i.e., entire matrix multiplication can be performed in the mappers.
        boolean needAgg = requiresAggregation(method);
        SparkAggType aggtype = getSparkMMAggregationType(needAgg);
        _outputEmptyBlocks = !OptimizerUtils.allowsToFilterEmptyBlockOutputs(this);
        // core matrix mult
        mapmult = new MapMult(getInput().get(0).constructLops(), getInput().get(1).constructLops(), getDataType(), getValueType(), (method == MMultMethod.MAPMM_R), false, _outputEmptyBlocks, aggtype);
    }
    setOutputDimensions(mapmult);
    setLineNumbers(mapmult);
    setLops(mapmult);
}

Example 59 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class AggBinaryOp method constructCPLopsMMWithLeftTransposeRewrite.

private Lop constructCPLopsMMWithLeftTransposeRewrite() {
    // guaranteed to exists
    Hop X = getInput().get(0).getInput().get(0);
    Hop Y = getInput().get(1);
    int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads);
    // right vector transpose
    Lop lY = Y.constructLops();
    Lop tY = (lY instanceof Transform && ((Transform) lY).getOperationType() == OperationTypes.Transpose) ? // if input is already a transpose, avoid redundant transpose ops
    lY.getInputs().get(0) : new Transform(lY, OperationTypes.Transpose, getDataType(), getValueType(), ExecType.CP, k);
    tY.getOutputParameters().setDimensions(Y.getDim2(), Y.getDim1(), getRowsInBlock(), getColsInBlock(), Y.getNnz());
    setLineNumbers(tY);
    // matrix mult
    Lop mult = new Binary(tY, X.constructLops(), Binary.OperationTypes.MATMULT, getDataType(), getValueType(), ExecType.CP, k);
    mult.getOutputParameters().setDimensions(Y.getDim2(), X.getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
    setLineNumbers(mult);
    // result transpose (dimensions set outside)
    Lop out = new Transform(mult, OperationTypes.Transpose, getDataType(), getValueType(), ExecType.CP, k);
    return out;
}

Example 60 with Lop

use of org.apache.sysml.lops.Lop in project incubator-systemml by apache.

the class AggBinaryOp method constructMRLopsMapMM.

// ////////////////////////
// MR Lops generation
// ///////////////////////
private void constructMRLopsMapMM(MMultMethod method) {
    if (method == MMultMethod.MAPMM_R && isLeftTransposeRewriteApplicable(false, true)) {
        setLops(constructMRLopsMapMMWithLeftTransposeRewrite());
    } else // GENERAL CASE
    {
        // If number of columns is smaller than block size then explicit aggregation is not required.
        // i.e., entire matrix multiplication can be performed in the mappers.
        boolean needAgg = requiresAggregation(method);
        boolean needPart = requiresPartitioning(method, false);
        _outputEmptyBlocks = !OptimizerUtils.allowsToFilterEmptyBlockOutputs(this);
        // pre partitioning
        Lop leftInput = getInput().get(0).constructLops();
        Lop rightInput = getInput().get(1).constructLops();
        if (needPart) {
            if (// left in distributed cache
            (method == MMultMethod.MAPMM_L)) {
                Hop input = getInput().get(0);
                ExecType etPart = (OptimizerUtils.estimateSizeExactSparsity(input.getDim1(), input.getDim2(), OptimizerUtils.getSparsity(input.getDim1(), input.getDim2(), input.getNnz())) < OptimizerUtils.getLocalMemBudget()) ? ExecType.CP : // operator selection
                ExecType.MR;
                leftInput = new DataPartition(input.constructLops(), DataType.MATRIX, ValueType.DOUBLE, etPart, PDataPartitionFormat.COLUMN_BLOCK_WISE_N);
                leftInput.getOutputParameters().setDimensions(input.getDim1(), input.getDim2(), getRowsInBlock(), getColsInBlock(), input.getNnz());
                setLineNumbers(leftInput);
            } else // right side in distributed cache
            {
                Hop input = getInput().get(1);
                ExecType etPart = (OptimizerUtils.estimateSizeExactSparsity(input.getDim1(), input.getDim2(), OptimizerUtils.getSparsity(input.getDim1(), input.getDim2(), input.getNnz())) < OptimizerUtils.getLocalMemBudget()) ? ExecType.CP : // operator selection
                ExecType.MR;
                rightInput = new DataPartition(input.constructLops(), DataType.MATRIX, ValueType.DOUBLE, etPart, PDataPartitionFormat.ROW_BLOCK_WISE_N);
                rightInput.getOutputParameters().setDimensions(input.getDim1(), input.getDim2(), getRowsInBlock(), getColsInBlock(), input.getNnz());
                setLineNumbers(rightInput);
            }
        }
        // core matrix mult
        MapMult mapmult = new MapMult(leftInput, rightInput, getDataType(), getValueType(), (method == MMultMethod.MAPMM_R), needPart, _outputEmptyBlocks);
        mapmult.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
        setLineNumbers(mapmult);
        // post aggregation
        if (needAgg) {
            Group grp = new Group(mapmult, Group.OperationTypes.Sort, getDataType(), getValueType());
            Aggregate agg1 = new Aggregate(grp, HopsAgg2Lops.get(outerOp), getDataType(), getValueType(), ExecType.MR);
            grp.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
            agg1.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
            setLineNumbers(agg1);
            // aggregation uses kahanSum but the inputs do not have correction values
            agg1.setupCorrectionLocation(CorrectionLocationType.NONE);
            setLops(agg1);
        } else {
            setLops(mapmult);
        }
    }
}