Examples with ExecType org.apache.sysml.lops.LopProperties.ExecType used on opensource projects

Example 11 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class UnaryOp method constructLops.

@Override
public Lop constructLops() throws HopsException, LopsException {
    //reuse existing lop
    if (getLops() != null)
        return getLops();
    try {
        Hop input = getInput().get(0);
        if (//value type casts or matrix to scalar
        getDataType() == DataType.SCALAR || (_op == OpOp1.CAST_AS_MATRIX && getInput().get(0).getDataType() == DataType.SCALAR) || (_op == OpOp1.CAST_AS_FRAME && getInput().get(0).getDataType() == DataType.SCALAR)) {
            if (//special handling IQM
            _op == Hop.OpOp1.IQM) {
                Lop iqmLop = constructLopsIQM();
                setLops(iqmLop);
            } else if (_op == Hop.OpOp1.MEDIAN) {
                Lop medianLop = constructLopsMedian();
                setLops(medianLop);
            } else //general case SCALAR/CAST (always in CP)
            {
                UnaryCP.OperationTypes optype = HopsOpOp1LopsUS.get(_op);
                if (optype == null)
                    throw new HopsException("Unknown UnaryCP lop type for UnaryOp operation type '" + _op + "'");
                UnaryCP unary1 = null;
                if ((_op == Hop.OpOp1.NROW || _op == Hop.OpOp1.NCOL || _op == Hop.OpOp1.LENGTH) && input instanceof UnaryOp && ((UnaryOp) input).getOp() == OpOp1.SELP) {
                    // Dimensions does not change during sel+ operation.
                    // This case is helpful to avoid unnecessary sel+ operation for fused maxpooling.
                    unary1 = new UnaryCP(input.getInput().get(0).constructLops(), optype, getDataType(), getValueType());
                } else
                    unary1 = new UnaryCP(input.constructLops(), optype, getDataType(), getValueType());
                setOutputDimensions(unary1);
                setLineNumbers(unary1);
                setLops(unary1);
            }
        } else //general case MATRIX
        {
            ExecType et = optFindExecType();
            //special handling cumsum/cumprod/cummin/cumsum
            if (isCumulativeUnaryOperation() && et != ExecType.CP) {
                //TODO additional physical operation if offsets fit in memory
                Lop cumsumLop = null;
                if (et == ExecType.MR)
                    cumsumLop = constructLopsMRCumulativeUnary();
                else
                    cumsumLop = constructLopsSparkCumulativeUnary();
                setLops(cumsumLop);
            } else //default unary 
            {
                int k = isCumulativeUnaryOperation() ? OptimizerUtils.getConstrainedNumThreads(_maxNumThreads) : 1;
                switch(_op) {
                    case SELP:
                    case EXP:
                    case SQRT:
                    case LOG:
                    case ABS:
                    case ROUND:
                    case FLOOR:
                    case CEIL:
                    case SIN:
                    case COS:
                    case TAN:
                    case ASIN:
                    case ACOS:
                    case ATAN:
                    case SIGN:
                        et = findGPUExecTypeByMemEstimate(et);
                        break;
                    default:
                }
                Unary unary1 = new Unary(input.constructLops(), HopsOpOp1LopsU.get(_op), getDataType(), getValueType(), et, k);
                setOutputDimensions(unary1);
                setLineNumbers(unary1);
                setLops(unary1);
            }
        }
    } catch (Exception e) {
        throw new HopsException(this.printErrorLocation() + "error constructing Lops for UnaryOp Hop -- \n ", e);
    }
    //add reblock/checkpoint lops if necessary
    constructAndSetLopsDataFlowProperties();
    return getLops();
}

Example 12 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class TernaryOp method constructLopsQuantile.

/**
	 * Method to construct LOPs when op = QUANTILE | INTERQUANTILE.
	 * 
	 * @throws HopsException if HopsException occurs
	 * @throws LopsException if LopsException occurs
	 */
private void constructLopsQuantile() throws HopsException, LopsException {
    if (_op != OpOp3.QUANTILE && _op != OpOp3.INTERQUANTILE)
        throw new HopsException("Unexpected operation: " + _op + ", expecting " + OpOp3.QUANTILE + " or " + OpOp3.INTERQUANTILE);
    ExecType et = optFindExecType();
    if (et == ExecType.MR) {
        CombineBinary combine = CombineBinary.constructCombineLop(OperationTypes.PreSort, getInput().get(0).constructLops(), getInput().get(1).constructLops(), DataType.MATRIX, getValueType());
        SortKeys sort = SortKeys.constructSortByValueLop(combine, SortKeys.OperationTypes.WithWeights, DataType.MATRIX, getValueType(), et);
        // If only a single quantile is computed, then "pick" operation executes in CP.
        ExecType et_pick = (getInput().get(2).getDataType() == DataType.SCALAR ? ExecType.CP : ExecType.MR);
        PickByCount pick = new PickByCount(sort, getInput().get(2).constructLops(), getDataType(), getValueType(), (_op == Hop.OpOp3.QUANTILE) ? PickByCount.OperationTypes.VALUEPICK : PickByCount.OperationTypes.RANGEPICK, et_pick, false);
        combine.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getInput().get(0).getRowsInBlock(), getInput().get(0).getColsInBlock(), getInput().get(0).getNnz());
        sort.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getInput().get(0).getRowsInBlock(), getInput().get(0).getColsInBlock(), getInput().get(0).getNnz());
        setOutputDimensions(pick);
        setLineNumbers(pick);
        setLops(pick);
    } else //CP/Spark 
    {
        SortKeys sort = SortKeys.constructSortByValueLop(getInput().get(0).constructLops(), getInput().get(1).constructLops(), SortKeys.OperationTypes.WithWeights, getInput().get(0).getDataType(), getInput().get(0).getValueType(), et);
        PickByCount pick = new PickByCount(sort, getInput().get(2).constructLops(), getDataType(), getValueType(), (_op == Hop.OpOp3.QUANTILE) ? PickByCount.OperationTypes.VALUEPICK : PickByCount.OperationTypes.RANGEPICK, et, true);
        sort.getOutputParameters().setDimensions(getInput().get(0).getDim1(), getInput().get(0).getDim2(), getInput().get(0).getRowsInBlock(), getInput().get(0).getColsInBlock(), getInput().get(0).getNnz());
        setOutputDimensions(pick);
        setLineNumbers(pick);
        setLops(pick);
    }
}

Example 13 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class TernaryOp method constructLopsPlusMult.

private void constructLopsPlusMult() throws HopsException, LopsException {
    if (_op != OpOp3.PLUS_MULT && _op != OpOp3.MINUS_MULT)
        throw new HopsException("Unexpected operation: " + _op + ", expecting " + OpOp3.PLUS_MULT + " or" + OpOp3.MINUS_MULT);
    ExecType et = null;
    if (DMLScript.USE_ACCELERATOR && (DMLScript.FORCE_ACCELERATOR || getMemEstimate() < OptimizerUtils.GPU_MEMORY_BUDGET))
        et = ExecType.GPU;
    else
        et = optFindExecType();
    PlusMult plusmult = null;
    if (et == ExecType.CP || et == ExecType.SPARK || et == ExecType.GPU) {
        plusmult = new PlusMult(getInput().get(0).constructLops(), getInput().get(1).constructLops(), getInput().get(2).constructLops(), _op, getDataType(), getValueType(), et);
    } else {
        //MR
        Hop left = getInput().get(0);
        Hop right = getInput().get(2);
        boolean requiresRep = BinaryOp.requiresReplication(left, right);
        Lop rightLop = right.constructLops();
        if (requiresRep) {
            //ncol of left input (determines num replicates)
            Lop offset = createOffsetLop(left, (right.getDim2() <= 1));
            rightLop = new RepMat(rightLop, offset, (right.getDim2() <= 1), right.getDataType(), right.getValueType());
            setOutputDimensions(rightLop);
            setLineNumbers(rightLop);
        }
        Group group1 = new Group(left.constructLops(), Group.OperationTypes.Sort, getDataType(), getValueType());
        setLineNumbers(group1);
        setOutputDimensions(group1);
        Group group2 = new Group(rightLop, Group.OperationTypes.Sort, getDataType(), getValueType());
        setLineNumbers(group2);
        setOutputDimensions(group2);
        plusmult = new PlusMult(group1, getInput().get(1).constructLops(), group2, _op, getDataType(), getValueType(), et);
    }
    setOutputDimensions(plusmult);
    setLineNumbers(plusmult);
    setLops(plusmult);
}

Example 14 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class TernaryOp method optFindExecType.

@Override
protected ExecType optFindExecType() throws HopsException {
    checkAndSetForcedPlatform();
    ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
    if (_etypeForced != null) {
        _etype = _etypeForced;
    } else {
        if (OptimizerUtils.isMemoryBasedOptLevel()) {
            _etype = findExecTypeByMemEstimate();
        } else if ((getInput().get(0).areDimsBelowThreshold() && getInput().get(1).areDimsBelowThreshold() && getInput().get(2).areDimsBelowThreshold()))
            //|| (getInput().get(0).isVector() && getInput().get(1).isVector() && getInput().get(1).isVector() )
            _etype = ExecType.CP;
        else
            _etype = REMOTE;
        //check for valid CP dimensions and matrix size
        checkAndSetInvalidCPDimsAndSize();
    }
    // dimension inputs are provided (and those values are unknown at initial compile time).
    if (ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true)) {
        if (_etype == REMOTE || (_etype == ExecType.CP && _dimInputsPresent))
            setRequiresRecompile();
    }
    return _etype;
}

Example 15 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class BinaryOp method optFindExecType.

@Override
protected ExecType optFindExecType() throws HopsException {
    checkAndSetForcedPlatform();
    ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
    DataType dt1 = getInput().get(0).getDataType();
    DataType dt2 = getInput().get(1).getDataType();
    if (_etypeForced != null) {
        _etype = _etypeForced;
    } else {
        if (OptimizerUtils.isMemoryBasedOptLevel()) {
            _etype = findExecTypeByMemEstimate();
        } else {
            _etype = null;
            if (dt1 == DataType.MATRIX && dt2 == DataType.MATRIX) {
                // OR if both are vectors
                if ((getInput().get(0).areDimsBelowThreshold() && getInput().get(1).areDimsBelowThreshold()) || (getInput().get(0).isVector() && getInput().get(1).isVector())) {
                    _etype = ExecType.CP;
                }
            } else if (dt1 == DataType.MATRIX && dt2 == DataType.SCALAR) {
                if (getInput().get(0).areDimsBelowThreshold() || getInput().get(0).isVector()) {
                    _etype = ExecType.CP;
                }
            } else if (dt1 == DataType.SCALAR && dt2 == DataType.MATRIX) {
                if (getInput().get(1).areDimsBelowThreshold() || getInput().get(1).isVector()) {
                    _etype = ExecType.CP;
                }
            } else {
                _etype = ExecType.CP;
            }
            //if no CP condition applied
            if (_etype == null)
                _etype = REMOTE;
        }
        //check for valid CP dimensions and matrix size
        checkAndSetInvalidCPDimsAndSize();
    }
    //single parent also in spark because it's likely cheap and reduces intermediates)
    if (_etype == ExecType.CP && _etypeForced != ExecType.CP && getDataType().isMatrix() && (dt1.isScalar() || dt2.isScalar()) && //scalar operations
    supportsMatrixScalarOperations() && //input is not checkpoint
    !(getInput().get(dt1.isScalar() ? 1 : 0) instanceof DataOp) && //unary scalar is only parent
    getInput().get(dt1.isScalar() ? 1 : 0).getParent().size() == 1 && //single block triggered exec
    !HopRewriteUtils.isSingleBlock(getInput().get(dt1.isScalar() ? 1 : 0)) && getInput().get(dt1.isScalar() ? 1 : 0).optFindExecType() == ExecType.SPARK) {
        //pull unary scalar operation into spark 
        _etype = ExecType.SPARK;
    }
    //mark for recompile (forever)
    if (ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype == REMOTE) {
        setRequiresRecompile();
    }
    //ensure cp exec type for single-node operations
    if (op == OpOp2.SOLVE) {
        _etype = ExecType.CP;
    }
    return _etype;
}