Examples with ExecType org.apache.sysml.lops.LopProperties.ExecType used on opensource projects

Example 6 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class ParameterizedBuiltinOp method optFindExecType.

@Override
protected ExecType optFindExecType() throws HopsException {
    checkAndSetForcedPlatform();
    ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
    if (_etypeForced != null) {
        _etype = _etypeForced;
    } else {
        if (_op == ParamBuiltinOp.TRANSFORM) {
            // force remote, at runtime cp transform triggered for small files.
            return (_etype = REMOTE);
        }
        if (OptimizerUtils.isMemoryBasedOptLevel()) {
            _etype = findExecTypeByMemEstimate();
        } else if (_op == ParamBuiltinOp.GROUPEDAGG && this.getInput().get(0).areDimsBelowThreshold()) {
            _etype = ExecType.CP;
        } else {
            _etype = REMOTE;
        }
        //check for valid CP dimensions and matrix size
        checkAndSetInvalidCPDimsAndSize();
    }
    //force CP for in-memory only transform builtins
    if ((_op == ParamBuiltinOp.TRANSFORMAPPLY && REMOTE == ExecType.MR) || _op == ParamBuiltinOp.TRANSFORMDECODE && REMOTE == ExecType.MR || _op == ParamBuiltinOp.TRANSFORMMETA || _op == ParamBuiltinOp.TOSTRING || _op == ParamBuiltinOp.CDF || _op == ParamBuiltinOp.INVCDF) {
        _etype = ExecType.CP;
    }
    //mark for recompile (forever)
    if (ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype == REMOTE)
        setRequiresRecompile();
    return _etype;
}

Example 7 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class IndexingOp method constructLops.

@Override
public Lop constructLops() throws HopsException, LopsException {
    //return already created lops
    if (getLops() != null)
        return getLops();
    Hop input = getInput().get(0);
    //rewrite remove unnecessary right indexing
    if (dimsKnown() && input.dimsKnown() && getDim1() == input.getDim1() && getDim2() == input.getDim2() && !(getDim1() == 1 && getDim2() == 1)) {
        setLops(input.constructLops());
    } else //actual lop construction, incl operator selection 
    {
        try {
            ExecType et = optFindExecType();
            if (et == ExecType.MR) {
                IndexingMethod method = optFindIndexingMethod(_rowLowerEqualsUpper, _colLowerEqualsUpper, input._dim1, input._dim2, _dim1, _dim2);
                Lop dummy = Data.createLiteralLop(ValueType.INT, Integer.toString(-1));
                RangeBasedReIndex reindex = new RangeBasedReIndex(input.constructLops(), getInput().get(1).constructLops(), getInput().get(2).constructLops(), getInput().get(3).constructLops(), getInput().get(4).constructLops(), dummy, dummy, getDataType(), getValueType(), et);
                setOutputDimensions(reindex);
                setLineNumbers(reindex);
                if (method == IndexingMethod.MR_RIX) {
                    Group group1 = new Group(reindex, Group.OperationTypes.Sort, DataType.MATRIX, getValueType());
                    setOutputDimensions(group1);
                    setLineNumbers(group1);
                    Aggregate agg1 = new Aggregate(group1, Aggregate.OperationTypes.Sum, DataType.MATRIX, getValueType(), et);
                    setOutputDimensions(agg1);
                    setLineNumbers(agg1);
                    setLops(agg1);
                } else //method == IndexingMethod.MR_VRIX
                {
                    setLops(reindex);
                }
            } else if (et == ExecType.SPARK) {
                IndexingMethod method = optFindIndexingMethod(_rowLowerEqualsUpper, _colLowerEqualsUpper, input._dim1, input._dim2, _dim1, _dim2);
                SparkAggType aggtype = (method == IndexingMethod.MR_VRIX || isBlockAligned()) ? SparkAggType.NONE : SparkAggType.MULTI_BLOCK;
                Lop dummy = Data.createLiteralLop(ValueType.INT, Integer.toString(-1));
                RangeBasedReIndex reindex = new RangeBasedReIndex(input.constructLops(), getInput().get(1).constructLops(), getInput().get(2).constructLops(), getInput().get(3).constructLops(), getInput().get(4).constructLops(), dummy, dummy, getDataType(), getValueType(), aggtype, et);
                setOutputDimensions(reindex);
                setLineNumbers(reindex);
                setLops(reindex);
            } else //CP
            {
                Lop dummy = Data.createLiteralLop(ValueType.INT, Integer.toString(-1));
                RangeBasedReIndex reindex = new RangeBasedReIndex(input.constructLops(), getInput().get(1).constructLops(), getInput().get(2).constructLops(), getInput().get(3).constructLops(), getInput().get(4).constructLops(), dummy, dummy, getDataType(), getValueType(), et);
                setOutputDimensions(reindex);
                setLineNumbers(reindex);
                setLops(reindex);
            }
        } catch (Exception e) {
            throw new HopsException(this.printErrorLocation() + "In IndexingOp Hop, error constructing Lops ", e);
        }
    }
    //add reblock/checkpoint lops if necessary
    constructAndSetLopsDataFlowProperties();
    return getLops();
}

Example 8 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class DataOp method constructLops.

@Override
public Lop constructLops() throws HopsException, LopsException {
    //return already created lops
    if (getLops() != null)
        return getLops();
    ExecType et = optFindExecType();
    Lop l = null;
    // construct lops for all input parameters
    HashMap<String, Lop> inputLops = new HashMap<String, Lop>();
    for (Entry<String, Integer> cur : _paramIndexMap.entrySet()) {
        inputLops.put(cur.getKey(), getInput().get(cur.getValue()).constructLops());
    }
    // Create the lop
    switch(_dataop) {
        case TRANSIENTREAD:
            l = new Data(HopsData2Lops.get(_dataop), null, inputLops, getName(), null, getDataType(), getValueType(), true, getInputFormatType());
            setOutputDimensions(l);
            break;
        case PERSISTENTREAD:
            l = new Data(HopsData2Lops.get(_dataop), null, inputLops, getName(), null, getDataType(), getValueType(), false, getInputFormatType());
            l.getOutputParameters().setDimensions(getDim1(), getDim2(), _inRowsInBlock, _inColsInBlock, getNnz(), getUpdateType());
            break;
        case PERSISTENTWRITE:
            l = new Data(HopsData2Lops.get(_dataop), getInput().get(0).constructLops(), inputLops, getName(), null, getDataType(), getValueType(), false, getInputFormatType());
            ((Data) l).setExecType(et);
            setOutputDimensions(l);
            break;
        case TRANSIENTWRITE:
            l = new Data(HopsData2Lops.get(_dataop), getInput().get(0).constructLops(), inputLops, getName(), null, getDataType(), getValueType(), true, getInputFormatType());
            setOutputDimensions(l);
            break;
        case FUNCTIONOUTPUT:
            l = new Data(HopsData2Lops.get(_dataop), getInput().get(0).constructLops(), inputLops, getName(), null, getDataType(), getValueType(), true, getInputFormatType());
            ((Data) l).setExecType(et);
            setOutputDimensions(l);
            break;
        default:
            throw new LopsException("Invalid operation type for Data LOP: " + _dataop);
    }
    setLineNumbers(l);
    setLops(l);
    //add reblock/checkpoint lops if necessary
    constructAndSetLopsDataFlowProperties();
    return getLops();
}

Example 9 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class Hop method constructAndSetCompressionLopIfRequired.

private void constructAndSetCompressionLopIfRequired() throws HopsException {
    //determine execution type
    ExecType et = ExecType.CP;
    if (OptimizerUtils.isSparkExecutionMode() && getDataType() != DataType.SCALAR) {
        //persist and unpersist calls (4x the memory budget is conservative)
        if (OptimizerUtils.isHybridExecutionMode() && 2 * _outputMemEstimate < OptimizerUtils.getLocalMemBudget() || _etypeForced == ExecType.CP) {
            et = ExecType.CP;
        } else //default case
        {
            et = ExecType.SPARK;
        }
    }
    //add reblock lop to output if required
    if (_requiresCompression) {
        try {
            Lop compress = new Compression(getLops(), getDataType(), getValueType(), et);
            setOutputDimensions(compress);
            setLineNumbers(compress);
            setLops(compress);
        } catch (LopsException ex) {
            throw new HopsException(ex);
        }
    }
}

Example 10 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class Hop method findExecTypeByMemEstimate.

/**
	 * This method determines the execution type (CP, MR) based ONLY on the 
	 * estimated memory footprint required for this operation, which includes 
	 * memory for all inputs and the output represented by this Hop.
	 * 
	 * It is used when <code>OptimizationType = MEMORY_BASED</code>.
	 * This optimization schedules an operation to CP whenever inputs+output 
	 * fit in memory -- note that this decision MAY NOT be optimal in terms of 
	 * execution time.
	 * 
	 * @return execution type
	 */
protected ExecType findExecTypeByMemEstimate() {
    ExecType et = null;
    char c = ' ';
    if (getMemEstimate() < OptimizerUtils.getLocalMemBudget()) {
        et = ExecType.CP;
    } else {
        if (DMLScript.rtplatform == DMLScript.RUNTIME_PLATFORM.HYBRID)
            et = ExecType.MR;
        else if (DMLScript.rtplatform == DMLScript.RUNTIME_PLATFORM.HYBRID_SPARK)
            et = ExecType.SPARK;
        c = '*';
    }
    if (LOG.isDebugEnabled()) {
        String s = String.format("  %c %-5s %-8s (%s,%s)  %s", c, getHopID(), getOpString(), OptimizerUtils.toMB(_outputMemEstimate), OptimizerUtils.toMB(_memEstimate), et);
        //System.out.println(s);
        LOG.debug(s);
    }
    return et;
}