Examples with MapMult org.apache.sysml.lops.MapMult used on opensource projects

Example 6 with MapMult

use of org.apache.sysml.lops.MapMult in project incubator-systemml by apache.

the class AggBinaryOp method constructSparkLopsMapMM.

private void constructSparkLopsMapMM(MMultMethod method) {
    Lop mapmult = null;
    if (isLeftTransposeRewriteApplicable(false, false)) {
        mapmult = constructSparkLopsMapMMWithLeftTransposeRewrite();
    } else {
        // If number of columns is smaller than block size then explicit aggregation is not required.
        // i.e., entire matrix multiplication can be performed in the mappers.
        boolean needAgg = requiresAggregation(method);
        SparkAggType aggtype = getSparkMMAggregationType(needAgg);
        _outputEmptyBlocks = !OptimizerUtils.allowsToFilterEmptyBlockOutputs(this);
        // core matrix mult
        mapmult = new MapMult(getInput().get(0).constructLops(), getInput().get(1).constructLops(), getDataType(), getValueType(), (method == MMultMethod.MAPMM_R), false, _outputEmptyBlocks, aggtype);
    }
    setOutputDimensions(mapmult);
    setLineNumbers(mapmult);
    setLops(mapmult);
}

Example 7 with MapMult

use of org.apache.sysml.lops.MapMult in project incubator-systemml by apache.

the class AggBinaryOp method constructMRLopsMapMM.

// ////////////////////////
// MR Lops generation
// ///////////////////////
private void constructMRLopsMapMM(MMultMethod method) {
    if (method == MMultMethod.MAPMM_R && isLeftTransposeRewriteApplicable(false, true)) {
        setLops(constructMRLopsMapMMWithLeftTransposeRewrite());
    } else // GENERAL CASE
    {
        // If number of columns is smaller than block size then explicit aggregation is not required.
        // i.e., entire matrix multiplication can be performed in the mappers.
        boolean needAgg = requiresAggregation(method);
        boolean needPart = requiresPartitioning(method, false);
        _outputEmptyBlocks = !OptimizerUtils.allowsToFilterEmptyBlockOutputs(this);
        // pre partitioning
        Lop leftInput = getInput().get(0).constructLops();
        Lop rightInput = getInput().get(1).constructLops();
        if (needPart) {
            if (// left in distributed cache
            (method == MMultMethod.MAPMM_L)) {
                Hop input = getInput().get(0);
                ExecType etPart = (OptimizerUtils.estimateSizeExactSparsity(input.getDim1(), input.getDim2(), OptimizerUtils.getSparsity(input.getDim1(), input.getDim2(), input.getNnz())) < OptimizerUtils.getLocalMemBudget()) ? ExecType.CP : // operator selection
                ExecType.MR;
                leftInput = new DataPartition(input.constructLops(), DataType.MATRIX, ValueType.DOUBLE, etPart, PDataPartitionFormat.COLUMN_BLOCK_WISE_N);
                leftInput.getOutputParameters().setDimensions(input.getDim1(), input.getDim2(), getRowsInBlock(), getColsInBlock(), input.getNnz());
                setLineNumbers(leftInput);
            } else // right side in distributed cache
            {
                Hop input = getInput().get(1);
                ExecType etPart = (OptimizerUtils.estimateSizeExactSparsity(input.getDim1(), input.getDim2(), OptimizerUtils.getSparsity(input.getDim1(), input.getDim2(), input.getNnz())) < OptimizerUtils.getLocalMemBudget()) ? ExecType.CP : // operator selection
                ExecType.MR;
                rightInput = new DataPartition(input.constructLops(), DataType.MATRIX, ValueType.DOUBLE, etPart, PDataPartitionFormat.ROW_BLOCK_WISE_N);
                rightInput.getOutputParameters().setDimensions(input.getDim1(), input.getDim2(), getRowsInBlock(), getColsInBlock(), input.getNnz());
                setLineNumbers(rightInput);
            }
        }
        // core matrix mult
        MapMult mapmult = new MapMult(leftInput, rightInput, getDataType(), getValueType(), (method == MMultMethod.MAPMM_R), needPart, _outputEmptyBlocks);
        mapmult.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
        setLineNumbers(mapmult);
        // post aggregation
        if (needAgg) {
            Group grp = new Group(mapmult, Group.OperationTypes.Sort, getDataType(), getValueType());
            Aggregate agg1 = new Aggregate(grp, HopsAgg2Lops.get(outerOp), getDataType(), getValueType(), ExecType.MR);
            grp.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
            agg1.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
            setLineNumbers(agg1);
            // aggregation uses kahanSum but the inputs do not have correction values
            agg1.setupCorrectionLocation(CorrectionLocationType.NONE);
            setLops(agg1);
        } else {
            setLops(mapmult);
        }
    }
}

Example 8 with MapMult

use of org.apache.sysml.lops.MapMult in project systemml by apache.

the class AggBinaryOp method constructMRLopsMapMMWithLeftTransposeRewrite.

private Lop constructMRLopsMapMMWithLeftTransposeRewrite() {
    // guaranteed to exists
    Hop X = getInput().get(0).getInput().get(0);
    Hop Y = getInput().get(1);
    // right vector transpose CP
    Lop tY = new Transform(Y.constructLops(), OperationTypes.Transpose, getDataType(), getValueType(), ExecType.CP);
    tY.getOutputParameters().setDimensions(Y.getDim2(), Y.getDim1(), getRowsInBlock(), getColsInBlock(), Y.getNnz());
    setLineNumbers(tY);
    // matrix mult
    // If number of columns is smaller than block size then explicit aggregation is not required.
    // i.e., entire matrix multiplication can be performed in the mappers.
    boolean needAgg = (X.getDim1() <= 0 || X.getDim1() > X.getRowsInBlock());
    // R disregarding transpose rewrite
    boolean needPart = requiresPartitioning(MMultMethod.MAPMM_R, true);
    // pre partitioning
    Lop dcinput = null;
    if (needPart) {
        ExecType etPart = (OptimizerUtils.estimateSizeExactSparsity(Y.getDim2(), Y.getDim1(), OptimizerUtils.getSparsity(Y.getDim2(), Y.getDim1(), Y.getNnz())) < OptimizerUtils.getLocalMemBudget()) ? ExecType.CP : // operator selection
        ExecType.MR;
        dcinput = new DataPartition(tY, DataType.MATRIX, ValueType.DOUBLE, etPart, PDataPartitionFormat.COLUMN_BLOCK_WISE_N);
        dcinput.getOutputParameters().setDimensions(Y.getDim2(), Y.getDim1(), getRowsInBlock(), getColsInBlock(), Y.getNnz());
        setLineNumbers(dcinput);
    } else
        dcinput = tY;
    MapMult mapmult = new MapMult(dcinput, X.constructLops(), getDataType(), getValueType(), false, needPart, false);
    mapmult.getOutputParameters().setDimensions(Y.getDim2(), X.getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
    setLineNumbers(mapmult);
    // post aggregation
    Lop mult = null;
    if (needAgg) {
        Group grp = new Group(mapmult, Group.OperationTypes.Sort, getDataType(), getValueType());
        grp.getOutputParameters().setDimensions(Y.getDim2(), X.getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
        setLineNumbers(grp);
        Aggregate agg1 = new Aggregate(grp, HopsAgg2Lops.get(outerOp), getDataType(), getValueType(), ExecType.MR);
        agg1.getOutputParameters().setDimensions(Y.getDim2(), X.getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
        setLineNumbers(agg1);
        agg1.setupCorrectionLocation(CorrectionLocationType.NONE);
        mult = agg1;
    } else
        mult = mapmult;
    // result transpose CP
    Lop out = new Transform(mult, OperationTypes.Transpose, getDataType(), getValueType(), ExecType.CP);
    out.getOutputParameters().setDimensions(X.getDim2(), Y.getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
    return out;
}

Example 9 with MapMult

use of org.apache.sysml.lops.MapMult in project systemml by apache.

the class AggBinaryOp method constructSparkLopsMapMMWithLeftTransposeRewrite.

private Lop constructSparkLopsMapMMWithLeftTransposeRewrite() {
    // guaranteed to exists
    Hop X = getInput().get(0).getInput().get(0);
    Hop Y = getInput().get(1);
    // right vector transpose
    Lop tY = new Transform(Y.constructLops(), OperationTypes.Transpose, getDataType(), getValueType(), ExecType.CP);
    tY.getOutputParameters().setDimensions(Y.getDim2(), Y.getDim1(), getRowsInBlock(), getColsInBlock(), Y.getNnz());
    setLineNumbers(tY);
    // matrix mult spark
    boolean needAgg = requiresAggregation(MMultMethod.MAPMM_R);
    SparkAggType aggtype = getSparkMMAggregationType(needAgg);
    _outputEmptyBlocks = !OptimizerUtils.allowsToFilterEmptyBlockOutputs(this);
    Lop mult = new MapMult(tY, X.constructLops(), getDataType(), getValueType(), false, false, _outputEmptyBlocks, aggtype);
    mult.getOutputParameters().setDimensions(Y.getDim2(), X.getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
    setLineNumbers(mult);
    // result transpose (dimensions set outside)
    Lop out = new Transform(mult, OperationTypes.Transpose, getDataType(), getValueType(), ExecType.CP);
    return out;
}

Example 10 with MapMult

use of org.apache.sysml.lops.MapMult in project systemml by apache.

the class Dag method computeFootprintInMapper.

/**
 * Computes the memory footprint required to execute <code>node</code> in the mapper.
 * It is used only for those nodes that use inputs from distributed cache. The returned
 * value is utilized in limiting the number of instructions piggybacked onto a single GMR mapper.
 *
 * @param node low-level operator
 * @return memory footprint
 */
private static double computeFootprintInMapper(Lop node) {
    // Memory limits must be checked only for nodes that use distributed cache
    if (!node.usesDistributedCache())
        // default behavior
        return 0.0;
    OutputParameters in1dims = node.getInputs().get(0).getOutputParameters();
    OutputParameters in2dims = node.getInputs().get(1).getOutputParameters();
    double footprint = 0;
    if (node instanceof MapMult) {
        int dcInputIndex = node.distributedCacheInputIndex()[0];
        footprint = AggBinaryOp.getMapmmMemEstimate(in1dims.getNumRows(), in1dims.getNumCols(), in1dims.getRowsInBlock(), in1dims.getColsInBlock(), in1dims.getNnz(), in2dims.getNumRows(), in2dims.getNumCols(), in2dims.getRowsInBlock(), in2dims.getColsInBlock(), in2dims.getNnz(), dcInputIndex, false);
    } else if (node instanceof PMMJ) {
        int dcInputIndex = node.distributedCacheInputIndex()[0];
        footprint = AggBinaryOp.getMapmmMemEstimate(in1dims.getNumRows(), 1, in1dims.getRowsInBlock(), in1dims.getColsInBlock(), in1dims.getNnz(), in2dims.getNumRows(), in2dims.getNumCols(), in2dims.getRowsInBlock(), in2dims.getColsInBlock(), in2dims.getNnz(), dcInputIndex, true);
    } else if (node instanceof AppendM) {
        footprint = BinaryOp.footprintInMapper(in1dims.getNumRows(), in1dims.getNumCols(), in2dims.getNumRows(), in2dims.getNumCols(), in1dims.getRowsInBlock(), in1dims.getColsInBlock());
    } else if (node instanceof BinaryM) {
        footprint = BinaryOp.footprintInMapper(in1dims.getNumRows(), in1dims.getNumCols(), in2dims.getNumRows(), in2dims.getNumCols(), in1dims.getRowsInBlock(), in1dims.getColsInBlock());
    } else {
        // default behavior
        return 0.0;
    }
    return footprint;
}