Examples with ExecType org.apache.sysml.lops.LopProperties.ExecType used on opensource projects

Example 41 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class DataGenOp method optFindExecType.

@Override
protected ExecType optFindExecType() {
    checkAndSetForcedPlatform();
    ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
    if (_etypeForced != null)
        _etype = _etypeForced;
    else {
        if (OptimizerUtils.isMemoryBasedOptLevel()) {
            _etype = findExecTypeByMemEstimate();
        } else if (this.areDimsBelowThreshold() || this.isVector())
            _etype = ExecType.CP;
        else
            _etype = REMOTE;
        // check for valid CP dimensions and matrix size
        checkAndSetInvalidCPDimsAndSize();
    }
    // mark for recompile (forever)
    setRequiresRecompileIfNecessary();
    // similarly, sample is currently not supported in MR either
    if (_op == DataGenMethod.SINIT)
        _etype = ExecType.CP;
    // workaround until sample supported in MR
    if (_op == DataGenMethod.SAMPLE && _etype == ExecType.MR)
        _etype = ExecType.CP;
    return _etype;
}

Example 42 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class DataOp method optFindExecType.

@Override
protected ExecType optFindExecType() {
    // MB: find exec type has two meanings here: (1) for write it means the actual
    // exec type, while (2) for read it affects the recompilation decision as needed
    // for example for sum(X) where the memory consumption is solely determined by the DataOp
    ExecType letype = (OptimizerUtils.isMemoryBasedOptLevel()) ? findExecTypeByMemEstimate() : null;
    ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
    // NOTE: independent of etype executed in MR (piggybacked) if input to persistent write is MR
    if (_dataop == DataOpTypes.PERSISTENTWRITE || _dataop == DataOpTypes.TRANSIENTWRITE) {
        checkAndSetForcedPlatform();
        // additional check for write only
        if (getDataType() == DataType.SCALAR || (getDataType() == DataType.FRAME && REMOTE == ExecType.MR))
            _etypeForced = ExecType.CP;
        if (_etypeForced != null) {
            _etype = _etypeForced;
        } else {
            if (OptimizerUtils.isMemoryBasedOptLevel()) {
                _etype = letype;
            } else if (getInput().get(0).areDimsBelowThreshold()) {
                _etype = ExecType.CP;
            } else {
                _etype = REMOTE;
            }
            // check for valid CP dimensions and matrix size
            checkAndSetInvalidCPDimsAndSize();
        }
        // mark for recompile (forever)
        setRequiresRecompileIfNecessary();
    } else // READ
    {
        // mark for recompile (forever)
        if (ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && letype == REMOTE && (_recompileRead || _requiresCheckpoint)) {
            setRequiresRecompile();
        }
        _etype = letype;
    }
    return _etype;
}

Example 43 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class FunctionOp method constructLops.

@Override
public Lop constructLops() {
    // return already created lops
    if (getLops() != null)
        return getLops();
    ExecType et = optFindExecType();
    // construct input lops (recursive)
    ArrayList<Lop> tmp = new ArrayList<>();
    for (Hop in : getInput()) tmp.add(in.constructLops());
    // construct function call
    Lop fcall = _singleOutFun ? new FunctionCallCPSingle(tmp, _fnamespace, _fname, et) : new FunctionCallCP(tmp, _fnamespace, _fname, _outputs, _outputHops, et);
    setLineNumbers(fcall);
    setLops(fcall);
    return getLops();
}

Example 44 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class Hop method constructAndSetCheckpointLopIfRequired.

private void constructAndSetCheckpointLopIfRequired() {
    // determine execution type
    ExecType et = ExecType.CP;
    if (OptimizerUtils.isSparkExecutionMode() && getDataType() != DataType.SCALAR) {
        // (2) avoid unnecessary creation of spark context (incl executors)
        if ((OptimizerUtils.isHybridExecutionMode() && hasValidCPDimsAndSize() && !OptimizerUtils.exceedsCachingThreshold(getDim2(), _outputMemEstimate)) || _etypeForced == ExecType.CP) {
            et = ExecType.CP;
        } else // default case
        {
            et = ExecType.SPARK;
        }
    }
    // add checkpoint lop to output if required
    if (_requiresCheckpoint && et != ExecType.CP) {
        try {
            // investigate need for serialized storage of large sparse matrices
            // (compile- instead of runtime-level for better debugging)
            boolean serializedStorage = false;
            if (getDataType() == DataType.MATRIX && dimsKnown(true)) {
                double matrixPSize = OptimizerUtils.estimatePartitionedSizeExactSparsity(_dim1, _dim2, _rows_in_block, _cols_in_block, _nnz);
                double dataCache = SparkExecutionContext.getDataMemoryBudget(true, true);
                serializedStorage = MatrixBlock.evalSparseFormatInMemory(_dim1, _dim2, _nnz) && // sparse in-memory does not fit in agg mem
                matrixPSize > dataCache && (OptimizerUtils.getSparsity(_dim1, _dim2, _nnz) < MatrixBlock.ULTRA_SPARSITY_TURN_POINT || // ultra-sparse or sparse w/o csr
                !Checkpoint.CHECKPOINT_SPARSE_CSR);
            } else if (!dimsKnown(true)) {
                setRequiresRecompile();
            }
            // construct checkpoint w/ right storage level
            Lop input = getLops();
            Lop chkpoint = new Checkpoint(input, getDataType(), getValueType(), serializedStorage ? Checkpoint.getSerializeStorageLevelString() : Checkpoint.getDefaultStorageLevelString());
            setOutputDimensions(chkpoint);
            setLineNumbers(chkpoint);
            setLops(chkpoint);
        } catch (LopsException ex) {
            throw new HopsException(ex);
        }
    }
}

Example 45 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class Hop method constructAndSetReblockLopIfRequired.

private void constructAndSetReblockLopIfRequired() {
    // determine execution type
    ExecType et = ExecType.CP;
    if (DMLScript.rtplatform != RUNTIME_PLATFORM.SINGLE_NODE && !(getDataType() == DataType.SCALAR)) {
        et = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
    }
    // add reblock lop to output if required
    if (_requiresReblock && et != ExecType.CP) {
        Lop input = getLops();
        Lop reblock = null;
        try {
            if (// CSV
            this instanceof DataOp && ((DataOp) this).getDataOpType() == DataOpTypes.PERSISTENTREAD && ((DataOp) this).getInputFormatType() == FileFormatTypes.CSV) {
                reblock = new CSVReBlock(input, getRowsInBlock(), getColsInBlock(), getDataType(), getValueType(), et);
            } else // TEXT / MM / BINARYBLOCK / BINARYCELL
            {
                reblock = new ReBlock(input, getRowsInBlock(), getColsInBlock(), getDataType(), getValueType(), _outputEmptyBlocks, et);
            }
        } catch (LopsException ex) {
            throw new HopsException(ex);
        }
        setOutputDimensions(reblock);
        setLineNumbers(reblock);
        setLops(reblock);
    }
}