Search in sources :

Example 31 with Group

use of org.apache.sysml.lops.Group in project incubator-systemml by apache.

the class AggBinaryOp method constructMRLopsCPMMWithLeftTransposeRewrite.

private Lop constructMRLopsCPMMWithLeftTransposeRewrite() {
    // guaranteed to exists
    Hop X = getInput().get(0).getInput().get(0);
    Hop Y = getInput().get(1);
    // right vector transpose CP
    Lop tY = new Transform(Y.constructLops(), OperationTypes.Transpose, getDataType(), getValueType(), ExecType.CP);
    tY.getOutputParameters().setDimensions(Y.getDim2(), Y.getDim1(), getRowsInBlock(), getColsInBlock(), Y.getNnz());
    setLineNumbers(tY);
    // matrix multiply
    MMCJType type = getMMCJAggregationType(X, Y);
    MMCJ mmcj = new MMCJ(tY, X.constructLops(), getDataType(), getValueType(), type, ExecType.MR);
    setOutputDimensions(mmcj);
    setLineNumbers(mmcj);
    Group grp = new Group(mmcj, Group.OperationTypes.Sort, getDataType(), getValueType());
    setOutputDimensions(grp);
    setLineNumbers(grp);
    Aggregate agg1 = new Aggregate(grp, HopsAgg2Lops.get(outerOp), getDataType(), getValueType(), ExecType.MR);
    setOutputDimensions(agg1);
    setLineNumbers(agg1);
    // aggregation uses kahanSum but the inputs do not have correction values
    agg1.setupCorrectionLocation(CorrectionLocationType.NONE);
    // result transpose CP
    Lop out = new Transform(agg1, OperationTypes.Transpose, getDataType(), getValueType(), ExecType.CP);
    out.getOutputParameters().setDimensions(X.getDim2(), Y.getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
    return out;
}
Also used : Group(org.apache.sysml.lops.Group) MMCJ(org.apache.sysml.lops.MMCJ) MultiThreadedHop(org.apache.sysml.hops.Hop.MultiThreadedHop) MMCJType(org.apache.sysml.lops.MMCJ.MMCJType) Lop(org.apache.sysml.lops.Lop) Transform(org.apache.sysml.lops.Transform) Aggregate(org.apache.sysml.lops.Aggregate)

Example 32 with Group

use of org.apache.sysml.lops.Group in project incubator-systemml by apache.

the class BinaryOp method constructLopsIQM.

private void constructLopsIQM(ExecType et) {
    if (et == ExecType.MR) {
        CombineBinary combine = CombineBinary.constructCombineLop(OperationTypes.PreSort, (Lop) getInput().get(0).constructLops(), (Lop) getInput().get(1).constructLops(), DataType.MATRIX, getValueType());
        combine.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getInput().get(0).getRowsInBlock(), getInput().get(0).getColsInBlock(), getInput().get(0).getNnz());
        SortKeys sort = SortKeys.constructSortByValueLop(combine, SortKeys.OperationTypes.WithWeights, DataType.MATRIX, ValueType.DOUBLE, ExecType.MR);
        // Sort dimensions are same as the first input
        sort.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getInput().get(0).getRowsInBlock(), getInput().get(0).getColsInBlock(), getInput().get(0).getNnz());
        Data lit = Data.createLiteralLop(ValueType.DOUBLE, Double.toString(0.25));
        setLineNumbers(lit);
        PickByCount pick = new PickByCount(sort, lit, DataType.MATRIX, getValueType(), PickByCount.OperationTypes.RANGEPICK);
        pick.getOutputParameters().setDimensions(-1, -1, getRowsInBlock(), getColsInBlock(), -1);
        setLineNumbers(pick);
        PartialAggregate pagg = new PartialAggregate(pick, HopsAgg2Lops.get(Hop.AggOp.SUM), HopsDirection2Lops.get(Hop.Direction.RowCol), DataType.MATRIX, getValueType());
        setLineNumbers(pagg);
        // Set the dimensions of PartialAggregate LOP based on the
        // direction in which aggregation is performed
        pagg.setDimensionsBasedOnDirection(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock());
        Group group1 = new Group(pagg, Group.OperationTypes.Sort, DataType.MATRIX, getValueType());
        setOutputDimensions(group1);
        setLineNumbers(group1);
        Aggregate agg1 = new Aggregate(group1, HopsAgg2Lops.get(Hop.AggOp.SUM), DataType.MATRIX, getValueType(), ExecType.MR);
        setOutputDimensions(agg1);
        agg1.setupCorrectionLocation(pagg.getCorrectionLocation());
        setLineNumbers(agg1);
        UnaryCP unary1 = new UnaryCP(agg1, HopsOpOp1LopsUS.get(OpOp1.CAST_AS_SCALAR), DataType.SCALAR, getValueType());
        unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
        setLineNumbers(unary1);
        Unary iqm = new Unary(sort, unary1, Unary.OperationTypes.MR_IQM, DataType.SCALAR, ValueType.DOUBLE, ExecType.CP);
        iqm.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
        setLineNumbers(iqm);
        setLops(iqm);
    } else {
        SortKeys sort = SortKeys.constructSortByValueLop(getInput().get(0).constructLops(), getInput().get(1).constructLops(), SortKeys.OperationTypes.WithWeights, getInput().get(0).getDataType(), getInput().get(0).getValueType(), et);
        sort.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getInput().get(0).getRowsInBlock(), getInput().get(0).getColsInBlock(), getInput().get(0).getNnz());
        PickByCount pick = new PickByCount(sort, null, getDataType(), getValueType(), PickByCount.OperationTypes.IQM, et, true);
        setOutputDimensions(pick);
        setLineNumbers(pick);
        setLops(pick);
    }
}
Also used : PartialAggregate(org.apache.sysml.lops.PartialAggregate) CombineBinary(org.apache.sysml.lops.CombineBinary) SortKeys(org.apache.sysml.lops.SortKeys) Group(org.apache.sysml.lops.Group) PickByCount(org.apache.sysml.lops.PickByCount) Data(org.apache.sysml.lops.Data) PartialAggregate(org.apache.sysml.lops.PartialAggregate) Aggregate(org.apache.sysml.lops.Aggregate) Unary(org.apache.sysml.lops.Unary) CombineUnary(org.apache.sysml.lops.CombineUnary) UnaryCP(org.apache.sysml.lops.UnaryCP)

Example 33 with Group

use of org.apache.sysml.lops.Group in project incubator-systemml by apache.

the class BinaryOp method constructLopsBinaryDefault.

private void constructLopsBinaryDefault() {
    /* Default behavior for BinaryOp */
    // it depends on input data types
    DataType dt1 = getInput().get(0).getDataType();
    DataType dt2 = getInput().get(1).getDataType();
    if (dt1 == dt2 && dt1 == DataType.SCALAR) {
        // Both operands scalar
        BinaryScalar binScalar1 = new BinaryScalar(getInput().get(0).constructLops(), getInput().get(1).constructLops(), HopsOpOp2LopsBS.get(op), getDataType(), getValueType());
        binScalar1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
        setLineNumbers(binScalar1);
        setLops(binScalar1);
    } else if ((dt1 == DataType.MATRIX && dt2 == DataType.SCALAR) || (dt1 == DataType.SCALAR && dt2 == DataType.MATRIX)) {
        // One operand is Matrix and the other is scalar
        ExecType et = optFindExecType();
        // select specific operator implementations
        Unary.OperationTypes ot = null;
        Hop right = getInput().get(1);
        if (op == OpOp2.POW && right instanceof LiteralOp && ((LiteralOp) right).getDoubleValue() == 2.0)
            ot = Unary.OperationTypes.POW2;
        else if (op == OpOp2.MULT && right instanceof LiteralOp && ((LiteralOp) right).getDoubleValue() == 2.0)
            ot = Unary.OperationTypes.MULTIPLY2;
        else
            // general case
            ot = HopsOpOp2LopsU.get(op);
        Unary unary1 = new Unary(getInput().get(0).constructLops(), getInput().get(1).constructLops(), ot, getDataType(), getValueType(), et);
        setOutputDimensions(unary1);
        setLineNumbers(unary1);
        setLops(unary1);
    } else {
        // Both operands are Matrixes
        ExecType et = optFindExecType();
        boolean isGPUSoftmax = et == ExecType.GPU && op == Hop.OpOp2.DIV && getInput().get(0) instanceof UnaryOp && getInput().get(1) instanceof AggUnaryOp && ((UnaryOp) getInput().get(0)).getOp() == OpOp1.EXP && ((AggUnaryOp) getInput().get(1)).getOp() == AggOp.SUM && ((AggUnaryOp) getInput().get(1)).getDirection() == Direction.Row && getInput().get(0) == getInput().get(1).getInput().get(0);
        if (isGPUSoftmax) {
            UnaryCP softmax = new UnaryCP(getInput().get(0).getInput().get(0).constructLops(), UnaryCP.OperationTypes.SOFTMAX, getDataType(), getValueType(), et);
            setOutputDimensions(softmax);
            setLineNumbers(softmax);
            setLops(softmax);
        } else if (et == ExecType.CP || et == ExecType.GPU) {
            Lop binary = null;
            boolean isLeftXGt = (getInput().get(0) instanceof BinaryOp) && ((BinaryOp) getInput().get(0)).getOp() == OpOp2.GREATER;
            Hop potentialZero = isLeftXGt ? ((BinaryOp) getInput().get(0)).getInput().get(1) : null;
            boolean isLeftXGt0 = isLeftXGt && potentialZero != null && potentialZero instanceof LiteralOp && ((LiteralOp) potentialZero).getDoubleValue() == 0;
            if (op == OpOp2.MULT && isLeftXGt0 && !getInput().get(0).isVector() && !getInput().get(1).isVector() && getInput().get(0).dimsKnown() && getInput().get(1).dimsKnown()) {
                binary = new ConvolutionTransform(getInput().get(0).getInput().get(0).constructLops(), getInput().get(1).constructLops(), ConvolutionTransform.OperationTypes.RELU_BACKWARD, getDataType(), getValueType(), et, -1);
            } else
                binary = new Binary(getInput().get(0).constructLops(), getInput().get(1).constructLops(), HopsOpOp2LopsB.get(op), getDataType(), getValueType(), et);
            setOutputDimensions(binary);
            setLineNumbers(binary);
            setLops(binary);
        } else if (et == ExecType.SPARK) {
            Hop left = getInput().get(0);
            Hop right = getInput().get(1);
            MMBinaryMethod mbin = optFindMMBinaryMethodSpark(left, right);
            Lop binary = null;
            if (mbin == MMBinaryMethod.MR_BINARY_UAGG_CHAIN) {
                AggUnaryOp uRight = (AggUnaryOp) right;
                binary = new BinaryUAggChain(left.constructLops(), HopsOpOp2LopsB.get(op), HopsAgg2Lops.get(uRight.getOp()), HopsDirection2Lops.get(uRight.getDirection()), getDataType(), getValueType(), et);
            } else if (mbin == MMBinaryMethod.MR_BINARY_M) {
                boolean partitioned = false;
                boolean isColVector = (right.getDim2() == 1 && left.getDim1() == right.getDim1());
                binary = new BinaryM(left.constructLops(), right.constructLops(), HopsOpOp2LopsB.get(op), getDataType(), getValueType(), et, partitioned, isColVector);
            } else {
                binary = new Binary(left.constructLops(), right.constructLops(), HopsOpOp2LopsB.get(op), getDataType(), getValueType(), et);
            }
            setOutputDimensions(binary);
            setLineNumbers(binary);
            setLops(binary);
        } else // MR
        {
            Hop left = getInput().get(0);
            Hop right = getInput().get(1);
            MMBinaryMethod mbin = optFindMMBinaryMethod(left, right);
            if (mbin == MMBinaryMethod.MR_BINARY_M) {
                boolean needPart = requiresPartitioning(right);
                Lop dcInput = right.constructLops();
                if (needPart) {
                    // right side in distributed cache
                    ExecType etPart = (OptimizerUtils.estimateSizeExactSparsity(right.getDim1(), right.getDim2(), OptimizerUtils.getSparsity(right.getDim1(), right.getDim2(), right.getNnz())) < OptimizerUtils.getLocalMemBudget()) ? ExecType.CP : // operator selection
                    ExecType.MR;
                    dcInput = new DataPartition(dcInput, DataType.MATRIX, ValueType.DOUBLE, etPart, (right.getDim2() == 1) ? PDataPartitionFormat.ROW_BLOCK_WISE_N : PDataPartitionFormat.COLUMN_BLOCK_WISE_N);
                    dcInput.getOutputParameters().setDimensions(right.getDim1(), right.getDim2(), right.getRowsInBlock(), right.getColsInBlock(), right.getNnz());
                    dcInput.setAllPositions(right.getFilename(), right.getBeginLine(), right.getBeginColumn(), right.getEndLine(), right.getEndColumn());
                }
                BinaryM binary = new BinaryM(left.constructLops(), dcInput, HopsOpOp2LopsB.get(op), getDataType(), getValueType(), ExecType.MR, needPart, (right.getDim2() == 1 && left.getDim1() == right.getDim1()));
                setOutputDimensions(binary);
                setLineNumbers(binary);
                setLops(binary);
            } else if (mbin == MMBinaryMethod.MR_BINARY_UAGG_CHAIN) {
                AggUnaryOp uRight = (AggUnaryOp) right;
                BinaryUAggChain bin = new BinaryUAggChain(left.constructLops(), HopsOpOp2LopsB.get(op), HopsAgg2Lops.get(uRight.getOp()), HopsDirection2Lops.get(uRight.getDirection()), getDataType(), getValueType(), et);
                setOutputDimensions(bin);
                setLineNumbers(bin);
                setLops(bin);
            } else if (mbin == MMBinaryMethod.MR_BINARY_OUTER_R) {
                boolean requiresRepLeft = (!right.dimsKnown() || right.getDim2() > right.getColsInBlock());
                boolean requiresRepRight = (!left.dimsKnown() || left.getDim1() > right.getRowsInBlock());
                Lop leftLop = left.constructLops();
                Lop rightLop = right.constructLops();
                if (requiresRepLeft) {
                    // ncol of right determines rep of left
                    Lop offset = createOffsetLop(right, true);
                    leftLop = new RepMat(leftLop, offset, true, left.getDataType(), left.getValueType());
                    setOutputDimensions(leftLop);
                    setLineNumbers(leftLop);
                }
                if (requiresRepRight) {
                    // nrow of right determines rep of right
                    Lop offset = createOffsetLop(left, false);
                    rightLop = new RepMat(rightLop, offset, false, right.getDataType(), right.getValueType());
                    setOutputDimensions(rightLop);
                    setLineNumbers(rightLop);
                }
                Group group1 = new Group(leftLop, Group.OperationTypes.Sort, getDataType(), getValueType());
                setLineNumbers(group1);
                setOutputDimensions(group1);
                Group group2 = new Group(rightLop, Group.OperationTypes.Sort, getDataType(), getValueType());
                setLineNumbers(group2);
                setOutputDimensions(group2);
                Binary binary = new Binary(group1, group2, HopsOpOp2LopsB.get(op), getDataType(), getValueType(), et);
                setOutputDimensions(binary);
                setLineNumbers(binary);
                setLops(binary);
            } else // MMBinaryMethod.MR_BINARY_R
            {
                boolean requiresRep = requiresReplication(left, right);
                Lop rightLop = right.constructLops();
                if (requiresRep) {
                    // ncol of left input (determines num replicates)
                    Lop offset = createOffsetLop(left, (right.getDim2() <= 1));
                    rightLop = new RepMat(rightLop, offset, (right.getDim2() <= 1), right.getDataType(), right.getValueType());
                    setOutputDimensions(rightLop);
                    setLineNumbers(rightLop);
                }
                Group group1 = new Group(getInput().get(0).constructLops(), Group.OperationTypes.Sort, getDataType(), getValueType());
                setLineNumbers(group1);
                setOutputDimensions(group1);
                Group group2 = new Group(rightLop, Group.OperationTypes.Sort, getDataType(), getValueType());
                setLineNumbers(group2);
                setOutputDimensions(group2);
                Binary binary = new Binary(group1, group2, HopsOpOp2LopsB.get(op), getDataType(), getValueType(), et);
                setLineNumbers(binary);
                setOutputDimensions(binary);
                setLops(binary);
            }
        }
    }
}
Also used : Group(org.apache.sysml.lops.Group) BinaryM(org.apache.sysml.lops.BinaryM) BinaryUAggChain(org.apache.sysml.lops.BinaryUAggChain) Lop(org.apache.sysml.lops.Lop) Unary(org.apache.sysml.lops.Unary) CombineUnary(org.apache.sysml.lops.CombineUnary) UnaryCP(org.apache.sysml.lops.UnaryCP) RepMat(org.apache.sysml.lops.RepMat) OperationTypes(org.apache.sysml.lops.CombineBinary.OperationTypes) DataType(org.apache.sysml.parser.Expression.DataType) ExecType(org.apache.sysml.lops.LopProperties.ExecType) Binary(org.apache.sysml.lops.Binary) CombineBinary(org.apache.sysml.lops.CombineBinary) BinaryScalar(org.apache.sysml.lops.BinaryScalar) ConvolutionTransform(org.apache.sysml.lops.ConvolutionTransform) DataPartition(org.apache.sysml.lops.DataPartition)

Example 34 with Group

use of org.apache.sysml.lops.Group in project incubator-systemml by apache.

the class BinaryOp method constructMRAppendLop.

/**
 * General case binary append.
 *
 * @param left high-level operator left
 * @param right high-level operator right
 * @param dt data type
 * @param vt value type
 * @param cbind true if cbind
 * @param current current high-level operator
 * @return low-level operator
 */
public static Lop constructMRAppendLop(Hop left, Hop right, DataType dt, ValueType vt, boolean cbind, Hop current) {
    Lop ret = null;
    long m1_dim1 = left.getDim1();
    long m1_dim2 = left.getDim2();
    long m2_dim1 = right.getDim1();
    long m2_dim2 = right.getDim2();
    // output rows
    long m3_dim1 = cbind ? m1_dim1 : ((m1_dim1 >= 0 && m2_dim1 >= 0) ? (m1_dim1 + m2_dim1) : -1);
    // output cols
    long m3_dim2 = cbind ? ((m1_dim2 >= 0 && m2_dim2 >= 0) ? (m1_dim2 + m2_dim2) : -1) : m1_dim2;
    // output nnz
    long m3_nnz = (left.getNnz() > 0 && right.getNnz() > 0) ? (left.getNnz() + right.getNnz()) : -1;
    long brlen = left.getRowsInBlock();
    long bclen = left.getColsInBlock();
    // offset 1st input
    Lop offset = createOffsetLop(left, cbind);
    AppendMethod am = optFindAppendMethod(m1_dim1, m1_dim2, m2_dim1, m2_dim2, brlen, bclen, cbind);
    switch(am) {
        case // special case map-only append
        MR_MAPPEND:
            {
                boolean needPart = requiresPartitioning(right);
                // pre partitioning
                Lop dcInput = right.constructLops();
                if (needPart) {
                    // right side in distributed cache
                    ExecType etPart = (OptimizerUtils.estimateSizeExactSparsity(right.getDim1(), right.getDim2(), OptimizerUtils.getSparsity(right.getDim1(), right.getDim2(), right.getNnz())) < OptimizerUtils.getLocalMemBudget()) ? ExecType.CP : // operator selection
                    ExecType.MR;
                    dcInput = new DataPartition(dcInput, DataType.MATRIX, ValueType.DOUBLE, etPart, PDataPartitionFormat.ROW_BLOCK_WISE_N);
                    dcInput.getOutputParameters().setDimensions(right.getDim1(), right.getDim2(), right.getRowsInBlock(), right.getColsInBlock(), right.getNnz());
                    dcInput.setAllPositions(right.getFilename(), right.getBeginLine(), right.getBeginColumn(), right.getEndLine(), right.getEndColumn());
                }
                AppendM appM = new AppendM(left.constructLops(), dcInput, offset, dt, vt, cbind, needPart, ExecType.MR);
                appM.setAllPositions(current.getFilename(), current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
                appM.getOutputParameters().setDimensions(m3_dim1, m3_dim2, brlen, bclen, m3_nnz);
                ret = appM;
                break;
            }
        case // special case reduce append w/ one column block
        MR_RAPPEND:
            {
                // group
                Group group1 = new Group(left.constructLops(), Group.OperationTypes.Sort, DataType.MATRIX, vt);
                group1.getOutputParameters().setDimensions(m1_dim1, m1_dim2, brlen, bclen, left.getNnz());
                group1.setAllPositions(left.getFilename(), left.getBeginLine(), left.getBeginColumn(), left.getEndLine(), left.getEndColumn());
                Group group2 = new Group(right.constructLops(), Group.OperationTypes.Sort, DataType.MATRIX, vt);
                group1.getOutputParameters().setDimensions(m2_dim1, m2_dim2, brlen, bclen, right.getNnz());
                group1.setAllPositions(right.getFilename(), right.getBeginLine(), right.getBeginColumn(), right.getEndLine(), right.getEndColumn());
                AppendR appR = new AppendR(group1, group2, dt, vt, cbind, ExecType.MR);
                appR.getOutputParameters().setDimensions(m3_dim1, m3_dim2, brlen, bclen, m3_nnz);
                appR.setAllPositions(current.getFilename(), current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
                ret = appR;
                break;
            }
        case MR_GAPPEND:
            {
                // general case: map expand append, reduce aggregate
                // offset second input
                Lop offset2 = createOffsetLop(right, cbind);
                AppendG appG = new AppendG(left.constructLops(), right.constructLops(), offset, offset2, dt, vt, cbind, ExecType.MR);
                appG.getOutputParameters().setDimensions(m3_dim1, m3_dim2, brlen, bclen, m3_nnz);
                appG.setAllPositions(current.getFilename(), current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
                // group
                Group group1 = new Group(appG, Group.OperationTypes.Sort, DataType.MATRIX, vt);
                group1.getOutputParameters().setDimensions(m3_dim1, m3_dim2, brlen, bclen, m3_nnz);
                group1.setAllPositions(current.getFilename(), current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
                // aggregate
                Aggregate agg1 = new Aggregate(group1, Aggregate.OperationTypes.Sum, DataType.MATRIX, vt, ExecType.MR);
                agg1.getOutputParameters().setDimensions(m3_dim1, m3_dim2, brlen, bclen, m3_nnz);
                agg1.setAllPositions(current.getFilename(), current.getBeginLine(), current.getBeginColumn(), current.getEndLine(), current.getEndColumn());
                ret = agg1;
                break;
            }
        default:
            throw new HopsException("Invalid MR append method: " + am);
    }
    return ret;
}
Also used : AppendG(org.apache.sysml.lops.AppendG) Group(org.apache.sysml.lops.Group) AppendR(org.apache.sysml.lops.AppendR) ExecType(org.apache.sysml.lops.LopProperties.ExecType) Lop(org.apache.sysml.lops.Lop) PartialAggregate(org.apache.sysml.lops.PartialAggregate) Aggregate(org.apache.sysml.lops.Aggregate) DataPartition(org.apache.sysml.lops.DataPartition) AppendM(org.apache.sysml.lops.AppendM)

Example 35 with Group

use of org.apache.sysml.lops.Group in project incubator-systemml by apache.

the class LeftIndexingOp method constructLops.

@Override
public Lop constructLops() {
    // return already created lops
    if (getLops() != null)
        return getLops();
    try {
        ExecType et = optFindExecType();
        if (et == ExecType.MR) {
            // the right matrix is reindexed
            Lop top = getInput().get(2).constructLops();
            Lop bottom = getInput().get(3).constructLops();
            Lop left = getInput().get(4).constructLops();
            Lop right = getInput().get(5).constructLops();
            // right hand matrix
            Lop nrow = new UnaryCP(getInput().get(0).constructLops(), OperationTypes.NROW, DataType.SCALAR, ValueType.INT);
            Lop ncol = new UnaryCP(getInput().get(0).constructLops(), OperationTypes.NCOL, DataType.SCALAR, ValueType.INT);
            Lop rightInput = null;
            if (isRightHandSideScalar()) {
                // insert cast to matrix if necessary (for reuse MR runtime)
                rightInput = new UnaryCP(getInput().get(1).constructLops(), OperationTypes.CAST_AS_MATRIX, DataType.MATRIX, ValueType.DOUBLE);
                rightInput.getOutputParameters().setDimensions(1L, 1L, ConfigurationManager.getBlocksize(), ConfigurationManager.getBlocksize(), -1L);
            } else
                rightInput = getInput().get(1).constructLops();
            RightIndex reindex = new RightIndex(rightInput, top, bottom, left, right, nrow, ncol, getDataType(), getValueType(), et, true);
            reindex.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
            setLineNumbers(reindex);
            Group group1 = new Group(reindex, Group.OperationTypes.Sort, DataType.MATRIX, getValueType());
            group1.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
            setLineNumbers(group1);
            // the left matrix is zeroed out
            ZeroOut zeroout = new ZeroOut(getInput().get(0).constructLops(), top, bottom, left, right, getInput().get(0).getDim1(), getInput().get(0).getDim2(), getDataType(), getValueType(), et);
            zeroout.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
            setLineNumbers(zeroout);
            Group group2 = new Group(zeroout, Group.OperationTypes.Sort, DataType.MATRIX, getValueType());
            group2.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
            setLineNumbers(group2);
            Binary binary = new Binary(group1, group2, HopsOpOp2LopsB.get(Hop.OpOp2.PLUS), getDataType(), getValueType(), et);
            binary.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
            setLineNumbers(binary);
            setLops(binary);
        } else if (et == ExecType.SPARK) {
            Hop left = getInput().get(0);
            Hop right = getInput().get(1);
            LeftIndexingMethod method = getOptMethodLeftIndexingMethod(left.getDim1(), left.getDim2(), left.getRowsInBlock(), left.getColsInBlock(), left.getNnz(), right.getDim1(), right.getDim2(), right.getNnz(), right.getDataType());
            // insert cast to matrix if necessary (for reuse broadcast runtime)
            Lop rightInput = right.constructLops();
            if (isRightHandSideScalar()) {
                rightInput = new UnaryCP(rightInput, (left.getDataType() == DataType.MATRIX ? OperationTypes.CAST_AS_MATRIX : OperationTypes.CAST_AS_FRAME), left.getDataType(), right.getValueType());
                long bsize = ConfigurationManager.getBlocksize();
                rightInput.getOutputParameters().setDimensions(1, 1, bsize, bsize, -1);
            }
            LeftIndex leftIndexLop = new LeftIndex(left.constructLops(), rightInput, getInput().get(2).constructLops(), getInput().get(3).constructLops(), getInput().get(4).constructLops(), getInput().get(5).constructLops(), getDataType(), getValueType(), et, getSpLixCacheType(method));
            setOutputDimensions(leftIndexLop);
            setLineNumbers(leftIndexLop);
            setLops(leftIndexLop);
        } else {
            LeftIndex left = new LeftIndex(getInput().get(0).constructLops(), getInput().get(1).constructLops(), getInput().get(2).constructLops(), getInput().get(3).constructLops(), getInput().get(4).constructLops(), getInput().get(5).constructLops(), getDataType(), getValueType(), et);
            setOutputDimensions(left);
            setLineNumbers(left);
            setLops(left);
        }
    } catch (Exception e) {
        throw new HopsException(this.printErrorLocation() + "In LeftIndexingOp Hop, error in constructing Lops ", e);
    }
    // add reblock/checkpoint lops if necessary
    constructAndSetLopsDataFlowProperties();
    return getLops();
}
Also used : Group(org.apache.sysml.lops.Group) ZeroOut(org.apache.sysml.lops.ZeroOut) RightIndex(org.apache.sysml.lops.RightIndex) LeftIndex(org.apache.sysml.lops.LeftIndex) ExecType(org.apache.sysml.lops.LopProperties.ExecType) Binary(org.apache.sysml.lops.Binary) Lop(org.apache.sysml.lops.Lop) UnaryCP(org.apache.sysml.lops.UnaryCP)

Aggregations

Group (org.apache.sysml.lops.Group)55 Lop (org.apache.sysml.lops.Lop)45 Aggregate (org.apache.sysml.lops.Aggregate)38 MultiThreadedHop (org.apache.sysml.hops.Hop.MultiThreadedHop)32 DataPartition (org.apache.sysml.lops.DataPartition)28 ExecType (org.apache.sysml.lops.LopProperties.ExecType)25 UnaryCP (org.apache.sysml.lops.UnaryCP)14 RepMat (org.apache.sysml.lops.RepMat)11 PartialAggregate (org.apache.sysml.lops.PartialAggregate)10 Unary (org.apache.sysml.lops.Unary)10 CombineUnary (org.apache.sysml.lops.CombineUnary)8 Transform (org.apache.sysml.lops.Transform)8 AppendR (org.apache.sysml.lops.AppendR)6 Data (org.apache.sysml.lops.Data)6 GroupedAggregate (org.apache.sysml.lops.GroupedAggregate)6 SortKeys (org.apache.sysml.lops.SortKeys)6 ArrayList (java.util.ArrayList)4 SparkAggType (org.apache.sysml.hops.AggBinaryOp.SparkAggType)4 OperationTypes (org.apache.sysml.lops.Aggregate.OperationTypes)4 Binary (org.apache.sysml.lops.Binary)4