Examples with ExecType org.apache.sysml.lops.LopProperties.ExecType used on opensource projects

Example 16 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class BinaryOp method constructLops.

@Override
public Lop constructLops() throws HopsException, LopsException {
    //return already created lops
    if (getLops() != null)
        return getLops();
    //select the execution type
    ExecType et = optFindExecType();
    switch(op) {
        case IQM:
            {
                constructLopsIQM(et);
                break;
            }
        case CENTRALMOMENT:
            {
                constructLopsCentralMoment(et);
                break;
            }
        case COVARIANCE:
            {
                constructLopsCovariance(et);
                break;
            }
        case QUANTILE:
        case INTERQUANTILE:
            {
                constructLopsQuantile(et);
                break;
            }
        case MEDIAN:
            {
                constructLopsMedian(et);
                break;
            }
        case CBIND:
        case RBIND:
            {
                constructLopsAppend(et);
                break;
            }
        default:
            constructLopsBinaryDefault();
    }
    //add reblock/checkpoint lops if necessary
    constructAndSetLopsDataFlowProperties();
    return getLops();
}

Example 17 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class AggBinaryOp method constructSparkLopsPMM.

private void constructSparkLopsPMM() throws HopsException, LopsException {
    //PMM has two potential modes (a) w/ full permutation matrix input, and 
    //(b) w/ already condensed input vector of target row positions.
    Hop pmInput = getInput().get(0);
    Hop rightInput = getInput().get(1);
    Lop lpmInput = pmInput.constructLops();
    Hop nrow = null;
    double mestPM = OptimizerUtils.estimateSize(pmInput.getDim1(), 1);
    ExecType etVect = (mestPM > OptimizerUtils.getLocalMemBudget()) ? ExecType.MR : ExecType.CP;
    //a) full permutation matrix input (potentially without empty block materialized)
    if (//not a vector
    pmInput.getDim2() != 1) {
        //compute condensed permutation matrix vector input			
        //v = rowMaxIndex(t(pm)) * rowMax(t(pm)) 
        ReorgOp transpose = HopRewriteUtils.createTranspose(pmInput);
        transpose.setForcedExecType(ExecType.SPARK);
        AggUnaryOp agg1 = HopRewriteUtils.createAggUnaryOp(transpose, AggOp.MAXINDEX, Direction.Row);
        agg1.setForcedExecType(ExecType.SPARK);
        AggUnaryOp agg2 = HopRewriteUtils.createAggUnaryOp(transpose, AggOp.MAX, Direction.Row);
        agg2.setForcedExecType(ExecType.SPARK);
        BinaryOp mult = HopRewriteUtils.createBinary(agg1, agg2, OpOp2.MULT);
        mult.setForcedExecType(ExecType.SPARK);
        //compute NROW target via nrow(m)
        nrow = HopRewriteUtils.createValueHop(pmInput, true);
        nrow.setOutputBlocksizes(0, 0);
        nrow.setForcedExecType(ExecType.CP);
        HopRewriteUtils.copyLineNumbers(this, nrow);
        lpmInput = mult.constructLops();
        HopRewriteUtils.removeChildReference(pmInput, transpose);
    } else //input vector
    {
        //compute NROW target via max(v)
        nrow = HopRewriteUtils.createAggUnaryOp(pmInput, AggOp.MAX, Direction.RowCol);
        nrow.setOutputBlocksizes(0, 0);
        nrow.setForcedExecType(etVect);
        HopRewriteUtils.copyLineNumbers(this, nrow);
    }
    //b) condensed permutation matrix vector input (target rows)
    _outputEmptyBlocks = !OptimizerUtils.allowsToFilterEmptyBlockOutputs(this);
    PMMJ pmm = new PMMJ(lpmInput, rightInput.constructLops(), nrow.constructLops(), getDataType(), getValueType(), false, _outputEmptyBlocks, ExecType.SPARK);
    setOutputDimensions(pmm);
    setLineNumbers(pmm);
    setLops(pmm);
    HopRewriteUtils.removeChildReference(pmInput, nrow);
}

Example 18 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class AggBinaryOp method constructMRLopsMapMMWithLeftTransposeRewrite.

private Lop constructMRLopsMapMMWithLeftTransposeRewrite() throws HopsException, LopsException {
    //guaranteed to exists
    Hop X = getInput().get(0).getInput().get(0);
    Hop Y = getInput().get(1);
    //right vector transpose CP
    Lop tY = new Transform(Y.constructLops(), OperationTypes.Transpose, getDataType(), getValueType(), ExecType.CP);
    tY.getOutputParameters().setDimensions(Y.getDim2(), Y.getDim1(), getRowsInBlock(), getColsInBlock(), Y.getNnz());
    setLineNumbers(tY);
    //matrix mult
    // If number of columns is smaller than block size then explicit aggregation is not required.
    // i.e., entire matrix multiplication can be performed in the mappers.
    boolean needAgg = (X.getDim1() <= 0 || X.getDim1() > X.getRowsInBlock());
    //R disregarding transpose rewrite
    boolean needPart = requiresPartitioning(MMultMethod.MAPMM_R, true);
    //pre partitioning
    Lop dcinput = null;
    if (needPart) {
        ExecType etPart = (OptimizerUtils.estimateSizeExactSparsity(Y.getDim2(), Y.getDim1(), OptimizerUtils.getSparsity(Y.getDim2(), Y.getDim1(), Y.getNnz())) < OptimizerUtils.getLocalMemBudget()) ? ExecType.CP : //operator selection
        ExecType.MR;
        dcinput = new DataPartition(tY, DataType.MATRIX, ValueType.DOUBLE, etPart, PDataPartitionFormat.COLUMN_BLOCK_WISE_N);
        dcinput.getOutputParameters().setDimensions(Y.getDim2(), Y.getDim1(), getRowsInBlock(), getColsInBlock(), Y.getNnz());
        setLineNumbers(dcinput);
    } else
        dcinput = tY;
    MapMult mapmult = new MapMult(dcinput, X.constructLops(), getDataType(), getValueType(), false, needPart, false);
    mapmult.getOutputParameters().setDimensions(Y.getDim2(), X.getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
    setLineNumbers(mapmult);
    //post aggregation 
    Lop mult = null;
    if (needAgg) {
        Group grp = new Group(mapmult, Group.OperationTypes.Sort, getDataType(), getValueType());
        grp.getOutputParameters().setDimensions(Y.getDim2(), X.getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
        setLineNumbers(grp);
        Aggregate agg1 = new Aggregate(grp, HopsAgg2Lops.get(outerOp), getDataType(), getValueType(), ExecType.MR);
        agg1.getOutputParameters().setDimensions(Y.getDim2(), X.getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
        setLineNumbers(agg1);
        agg1.setupCorrectionLocation(CorrectionLocationType.NONE);
        mult = agg1;
    } else
        mult = mapmult;
    //result transpose CP 
    Lop out = new Transform(mult, OperationTypes.Transpose, getDataType(), getValueType(), ExecType.CP);
    out.getOutputParameters().setDimensions(X.getDim2(), Y.getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
    return out;
}

Example 19 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class AggUnaryOp method constructLops.

@Override
public Lop constructLops() throws HopsException, LopsException {
    //return already created lops
    if (getLops() != null)
        return getLops();
    try {
        ExecType et = optFindExecType();
        Hop input = getInput().get(0);
        if (et == ExecType.CP) {
            Lop agg1 = null;
            if (isTernaryAggregateRewriteApplicable(et)) {
                agg1 = constructLopsTernaryAggregateRewrite(et);
            } else if (isUnaryAggregateOuterCPRewriteApplicable()) {
                OperationTypes op = HopsAgg2Lops.get(_op);
                DirectionTypes dir = HopsDirection2Lops.get(_direction);
                BinaryOp binput = (BinaryOp) getInput().get(0);
                agg1 = new UAggOuterChain(binput.getInput().get(0).constructLops(), binput.getInput().get(1).constructLops(), op, dir, HopsOpOp2LopsB.get(binput.getOp()), DataType.MATRIX, getValueType(), ExecType.CP);
                PartialAggregate.setDimensionsBasedOnDirection(agg1, getDim1(), getDim2(), input.getRowsInBlock(), input.getColsInBlock(), dir);
                if (getDataType() == DataType.SCALAR) {
                    UnaryCP unary1 = new UnaryCP(agg1, HopsOpOp1LopsUS.get(OpOp1.CAST_AS_SCALAR), getDataType(), getValueType());
                    unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
                    setLineNumbers(unary1);
                    setLops(unary1);
                }
            } else {
                //general case		
                int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads);
                if (DMLScript.USE_ACCELERATOR && (DMLScript.FORCE_ACCELERATOR || getMemEstimate() < OptimizerUtils.GPU_MEMORY_BUDGET)) {
                    // Only implemented methods for GPU
                    if ((_op == AggOp.SUM && (_direction == Direction.RowCol || _direction == Direction.Row || _direction == Direction.Col)) || (_op == AggOp.SUM_SQ && (_direction == Direction.RowCol || _direction == Direction.Row || _direction == Direction.Col)) || (_op == AggOp.MAX && (_direction == Direction.RowCol || _direction == Direction.Row || _direction == Direction.Col)) || (_op == AggOp.MIN && (_direction == Direction.RowCol || _direction == Direction.Row || _direction == Direction.Col)) || (_op == AggOp.MEAN && (_direction == Direction.RowCol || _direction == Direction.Row || _direction == Direction.Col)) || (_op == AggOp.VAR && (_direction == Direction.RowCol || _direction == Direction.Row || _direction == Direction.Col)) || (_op == AggOp.PROD && (_direction == Direction.RowCol))) {
                        et = ExecType.GPU;
                        k = 1;
                    }
                }
                agg1 = new PartialAggregate(input.constructLops(), HopsAgg2Lops.get(_op), HopsDirection2Lops.get(_direction), getDataType(), getValueType(), et, k);
            }
            setOutputDimensions(agg1);
            setLineNumbers(agg1);
            setLops(agg1);
            if (getDataType() == DataType.SCALAR) {
                agg1.getOutputParameters().setDimensions(1, 1, getRowsInBlock(), getColsInBlock(), getNnz());
            }
        } else if (et == ExecType.MR) {
            OperationTypes op = HopsAgg2Lops.get(_op);
            DirectionTypes dir = HopsDirection2Lops.get(_direction);
            //unary aggregate operation
            Lop transform1 = null;
            if (isUnaryAggregateOuterRewriteApplicable()) {
                BinaryOp binput = (BinaryOp) getInput().get(0);
                transform1 = new UAggOuterChain(binput.getInput().get(0).constructLops(), binput.getInput().get(1).constructLops(), op, dir, HopsOpOp2LopsB.get(binput.getOp()), DataType.MATRIX, getValueType(), ExecType.MR);
                PartialAggregate.setDimensionsBasedOnDirection(transform1, getDim1(), getDim2(), input.getRowsInBlock(), input.getColsInBlock(), dir);
            } else //default
            {
                transform1 = new PartialAggregate(input.constructLops(), op, dir, DataType.MATRIX, getValueType());
                ((PartialAggregate) transform1).setDimensionsBasedOnDirection(getDim1(), getDim2(), input.getRowsInBlock(), input.getColsInBlock());
            }
            setLineNumbers(transform1);
            //aggregation if required
            Lop aggregate = null;
            Group group1 = null;
            Aggregate agg1 = null;
            if (requiresAggregation(input, _direction) || transform1 instanceof UAggOuterChain) {
                group1 = new Group(transform1, Group.OperationTypes.Sort, DataType.MATRIX, getValueType());
                group1.getOutputParameters().setDimensions(getDim1(), getDim2(), input.getRowsInBlock(), input.getColsInBlock(), getNnz());
                setLineNumbers(group1);
                agg1 = new Aggregate(group1, HopsAgg2Lops.get(_op), DataType.MATRIX, getValueType(), et);
                agg1.getOutputParameters().setDimensions(getDim1(), getDim2(), input.getRowsInBlock(), input.getColsInBlock(), getNnz());
                agg1.setupCorrectionLocation(PartialAggregate.getCorrectionLocation(op, dir));
                setLineNumbers(agg1);
                aggregate = agg1;
            } else {
                ((PartialAggregate) transform1).setDropCorrection();
                aggregate = transform1;
            }
            setLops(aggregate);
            //cast if required
            if (getDataType() == DataType.SCALAR) {
                // Set the dimensions of PartialAggregate LOP based on the
                // direction in which aggregation is performed
                PartialAggregate.setDimensionsBasedOnDirection(transform1, input.getDim1(), input.getDim2(), input.getRowsInBlock(), input.getColsInBlock(), dir);
                if (group1 != null && agg1 != null) {
                    //if aggregation required
                    group1.getOutputParameters().setDimensions(input.getDim1(), input.getDim2(), input.getRowsInBlock(), input.getColsInBlock(), getNnz());
                    agg1.getOutputParameters().setDimensions(1, 1, input.getRowsInBlock(), input.getColsInBlock(), getNnz());
                }
                UnaryCP unary1 = new UnaryCP(aggregate, HopsOpOp1LopsUS.get(OpOp1.CAST_AS_SCALAR), getDataType(), getValueType());
                unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
                setLineNumbers(unary1);
                setLops(unary1);
            }
        } else if (et == ExecType.SPARK) {
            OperationTypes op = HopsAgg2Lops.get(_op);
            DirectionTypes dir = HopsDirection2Lops.get(_direction);
            //unary aggregate
            if (isTernaryAggregateRewriteApplicable(et)) {
                Lop aggregate = constructLopsTernaryAggregateRewrite(et);
                //0x0 (scalar)
                setOutputDimensions(aggregate);
                setLineNumbers(aggregate);
                setLops(aggregate);
            } else if (isUnaryAggregateOuterSPRewriteApplicable()) {
                BinaryOp binput = (BinaryOp) getInput().get(0);
                Lop transform1 = new UAggOuterChain(binput.getInput().get(0).constructLops(), binput.getInput().get(1).constructLops(), op, dir, HopsOpOp2LopsB.get(binput.getOp()), DataType.MATRIX, getValueType(), ExecType.SPARK);
                PartialAggregate.setDimensionsBasedOnDirection(transform1, getDim1(), getDim2(), input.getRowsInBlock(), input.getColsInBlock(), dir);
                setLineNumbers(transform1);
                setLops(transform1);
                if (getDataType() == DataType.SCALAR) {
                    UnaryCP unary1 = new UnaryCP(transform1, HopsOpOp1LopsUS.get(OpOp1.CAST_AS_SCALAR), getDataType(), getValueType());
                    unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
                    setLineNumbers(unary1);
                    setLops(unary1);
                }
            } else //default
            {
                boolean needAgg = requiresAggregation(input, _direction);
                SparkAggType aggtype = getSparkUnaryAggregationType(needAgg);
                PartialAggregate aggregate = new PartialAggregate(input.constructLops(), HopsAgg2Lops.get(_op), HopsDirection2Lops.get(_direction), DataType.MATRIX, getValueType(), aggtype, et);
                aggregate.setDimensionsBasedOnDirection(getDim1(), getDim2(), input.getRowsInBlock(), input.getColsInBlock());
                setLineNumbers(aggregate);
                setLops(aggregate);
                if (getDataType() == DataType.SCALAR) {
                    UnaryCP unary1 = new UnaryCP(aggregate, HopsOpOp1LopsUS.get(OpOp1.CAST_AS_SCALAR), getDataType(), getValueType());
                    unary1.getOutputParameters().setDimensions(0, 0, 0, 0, -1);
                    setLineNumbers(unary1);
                    setLops(unary1);
                }
            }
        }
    } catch (Exception e) {
        throw new HopsException(this.printErrorLocation() + "In AggUnary Hop, error constructing Lops ", e);
    }
    //add reblock/checkpoint lops if necessary
    constructAndSetLopsDataFlowProperties();
    //return created lops
    return getLops();
}

Example 20 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class BinaryOp method constructLopsMedian.

private void constructLopsMedian(ExecType et) throws HopsException, LopsException {
    if (et == ExecType.MR) {
        CombineBinary combine = CombineBinary.constructCombineLop(OperationTypes.PreSort, getInput().get(0).constructLops(), getInput().get(1).constructLops(), DataType.MATRIX, getValueType());
        SortKeys sort = SortKeys.constructSortByValueLop(combine, SortKeys.OperationTypes.WithWeights, DataType.MATRIX, getValueType(), et);
        combine.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
        // Sort dimensions are same as the first input
        sort.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getInput().get(0).getRowsInBlock(), getInput().get(0).getColsInBlock(), getInput().get(0).getNnz());
        ExecType et_pick = ExecType.CP;
        PickByCount pick = new PickByCount(sort, Data.createLiteralLop(ValueType.DOUBLE, Double.toString(0.5)), getDataType(), getValueType(), PickByCount.OperationTypes.MEDIAN, et_pick, false);
        pick.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
        pick.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
        setLops(pick);
    } else {
        SortKeys sort = SortKeys.constructSortByValueLop(getInput().get(0).constructLops(), getInput().get(1).constructLops(), SortKeys.OperationTypes.WithWeights, getInput().get(0).getDataType(), getInput().get(0).getValueType(), et);
        sort.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getInput().get(0).getRowsInBlock(), getInput().get(0).getColsInBlock(), getInput().get(0).getNnz());
        PickByCount pick = new PickByCount(sort, Data.createLiteralLop(ValueType.DOUBLE, Double.toString(0.5)), getDataType(), getValueType(), PickByCount.OperationTypes.MEDIAN, et, true);
        pick.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
        pick.setAllPositions(this.getBeginLine(), this.getBeginColumn(), this.getEndLine(), this.getEndColumn());
        setLops(pick);
    }
}