Examples with ExecType org.apache.sysml.lops.LopProperties.ExecType used on opensource projects

Example 31 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class WeightedDivMMR method getInstructions.

/* CP/SPARK instruction generation */
@Override
public String getInstructions(String input1, String input2, String input3, String input4, String output) {
    StringBuilder sb = new StringBuilder();
    final ExecType et = getExecType();
    sb.append(et);
    sb.append(Lop.OPERAND_DELIMITOR);
    sb.append(OPCODE);
    sb.append(Lop.OPERAND_DELIMITOR);
    sb.append(getInputs().get(0).prepInputOperand(input1));
    sb.append(Lop.OPERAND_DELIMITOR);
    sb.append(getInputs().get(1).prepInputOperand(input2));
    sb.append(Lop.OPERAND_DELIMITOR);
    sb.append(getInputs().get(2).prepInputOperand(input3));
    sb.append(Lop.OPERAND_DELIMITOR);
    if ((et == ExecType.MR) && (getInputs().get(3).getDataType() == DataType.SCALAR)) {
        sb.append(getInputs().get(3).prepScalarInputOperand(et));
    } else {
        sb.append(getInputs().get(3).prepInputOperand(input4));
    }
    sb.append(Lop.OPERAND_DELIMITOR);
    sb.append(prepOutputOperand(output));
    sb.append(Lop.OPERAND_DELIMITOR);
    sb.append(_weightsType);
    sb.append(Lop.OPERAND_DELIMITOR);
    sb.append(_cacheU);
    sb.append(Lop.OPERAND_DELIMITOR);
    sb.append(_cacheV);
    return sb.toString();
}

Example 32 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class DataGen method getSeqInstructionCPSpark.

/**
 * Private method that generates CP Instruction for Seq.
 *
 * @param output output operand
 * @return cp instruction for seq
 */
private String getSeqInstructionCPSpark(String output) {
    if (method != DataGenMethod.SEQ)
        throw new LopsException("Invalid instruction generation for data generation method " + method);
    StringBuilder sb = new StringBuilder();
    ExecType et = getExecType();
    sb.append(et);
    sb.append(Lop.OPERAND_DELIMITOR);
    Lop iLop = null;
    iLop = _inputParams.get(Statement.SEQ_FROM.toString());
    String fromString = iLop.prepScalarInputOperand(et);
    iLop = _inputParams.get(Statement.SEQ_TO.toString());
    String toString = iLop.prepScalarInputOperand(et);
    iLop = _inputParams.get(Statement.SEQ_INCR.toString());
    String incrString = iLop.prepScalarInputOperand(et);
    String rowsString = String.valueOf(this.getOutputParameters().getNumRows());
    String colsString = String.valueOf(this.getOutputParameters().getNumCols());
    String rowsInBlockString = String.valueOf(this.getOutputParameters().getRowsInBlock());
    String colsInBlockString = String.valueOf(this.getOutputParameters().getColsInBlock());
    sb.append(DataGen.SEQ_OPCODE);
    sb.append(OPERAND_DELIMITOR);
    sb.append(rowsString);
    sb.append(OPERAND_DELIMITOR);
    sb.append(colsString);
    sb.append(OPERAND_DELIMITOR);
    sb.append(rowsInBlockString);
    sb.append(OPERAND_DELIMITOR);
    sb.append(colsInBlockString);
    sb.append(OPERAND_DELIMITOR);
    sb.append(fromString);
    sb.append(OPERAND_DELIMITOR);
    sb.append(toString);
    sb.append(OPERAND_DELIMITOR);
    sb.append(incrString);
    sb.append(OPERAND_DELIMITOR);
    if (et == ExecType.MR) {
        sb.append(baseDir);
        sb.append(OPERAND_DELIMITOR);
    }
    sb.append(this.prepOutputOperand(output));
    return sb.toString();
}

Example 33 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class AggBinaryOp method optFindExecType.

@Override
protected ExecType optFindExecType() {
    checkAndSetForcedPlatform();
    ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
    if (_etypeForced != null) {
        _etype = _etypeForced;
    } else {
        if (OptimizerUtils.isMemoryBasedOptLevel()) {
            _etype = findExecTypeByMemEstimate();
        } else // OR if it is vector-vector inner product
        if ((getInput().get(0).areDimsBelowThreshold() && getInput().get(1).areDimsBelowThreshold()) || (getInput().get(0).isVector() && getInput().get(1).isVector() && !isOuterProduct())) {
            _etype = ExecType.CP;
        } else {
            _etype = REMOTE;
        }
        // check for valid CP mmchain, send invalid memory requirements to remote
        if (_etype == ExecType.CP && checkMapMultChain() != ChainType.NONE && OptimizerUtils.getLocalMemBudget() < getInput().get(0).getInput().get(0).getOutputMemEstimate())
            _etype = REMOTE;
        // check for valid CP dimensions and matrix size
        checkAndSetInvalidCPDimsAndSize();
    }
    // single parent also in spark because it's likely cheap and reduces data transfer)
    if (_etype == ExecType.CP && _etypeForced != ExecType.CP && (isApplicableForTransitiveSparkExecType(true) || isApplicableForTransitiveSparkExecType(false))) {
        // pull binary aggregate into spark
        _etype = ExecType.SPARK;
    }
    // mark for recompile (forever)
    setRequiresRecompileIfNecessary();
    return _etype;
}

Example 34 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class AggBinaryOp method constructMRLopsMapMM.

// ////////////////////////
// MR Lops generation
// ///////////////////////
private void constructMRLopsMapMM(MMultMethod method) {
    if (method == MMultMethod.MAPMM_R && isLeftTransposeRewriteApplicable(false, true)) {
        setLops(constructMRLopsMapMMWithLeftTransposeRewrite());
    } else // GENERAL CASE
    {
        // If number of columns is smaller than block size then explicit aggregation is not required.
        // i.e., entire matrix multiplication can be performed in the mappers.
        boolean needAgg = requiresAggregation(method);
        boolean needPart = requiresPartitioning(method, false);
        _outputEmptyBlocks = !OptimizerUtils.allowsToFilterEmptyBlockOutputs(this);
        // pre partitioning
        Lop leftInput = getInput().get(0).constructLops();
        Lop rightInput = getInput().get(1).constructLops();
        if (needPart) {
            if (// left in distributed cache
            (method == MMultMethod.MAPMM_L)) {
                Hop input = getInput().get(0);
                ExecType etPart = (OptimizerUtils.estimateSizeExactSparsity(input.getDim1(), input.getDim2(), OptimizerUtils.getSparsity(input.getDim1(), input.getDim2(), input.getNnz())) < OptimizerUtils.getLocalMemBudget()) ? ExecType.CP : // operator selection
                ExecType.MR;
                leftInput = new DataPartition(input.constructLops(), DataType.MATRIX, ValueType.DOUBLE, etPart, PDataPartitionFormat.COLUMN_BLOCK_WISE_N);
                leftInput.getOutputParameters().setDimensions(input.getDim1(), input.getDim2(), getRowsInBlock(), getColsInBlock(), input.getNnz());
                setLineNumbers(leftInput);
            } else // right side in distributed cache
            {
                Hop input = getInput().get(1);
                ExecType etPart = (OptimizerUtils.estimateSizeExactSparsity(input.getDim1(), input.getDim2(), OptimizerUtils.getSparsity(input.getDim1(), input.getDim2(), input.getNnz())) < OptimizerUtils.getLocalMemBudget()) ? ExecType.CP : // operator selection
                ExecType.MR;
                rightInput = new DataPartition(input.constructLops(), DataType.MATRIX, ValueType.DOUBLE, etPart, PDataPartitionFormat.ROW_BLOCK_WISE_N);
                rightInput.getOutputParameters().setDimensions(input.getDim1(), input.getDim2(), getRowsInBlock(), getColsInBlock(), input.getNnz());
                setLineNumbers(rightInput);
            }
        }
        // core matrix mult
        MapMult mapmult = new MapMult(leftInput, rightInput, getDataType(), getValueType(), (method == MMultMethod.MAPMM_R), needPart, _outputEmptyBlocks);
        mapmult.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
        setLineNumbers(mapmult);
        // post aggregation
        if (needAgg) {
            Group grp = new Group(mapmult, Group.OperationTypes.Sort, getDataType(), getValueType());
            Aggregate agg1 = new Aggregate(grp, HopsAgg2Lops.get(outerOp), getDataType(), getValueType(), ExecType.MR);
            grp.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
            agg1.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
            setLineNumbers(agg1);
            // aggregation uses kahanSum but the inputs do not have correction values
            agg1.setupCorrectionLocation(CorrectionLocationType.NONE);
            setLops(agg1);
        } else {
            setLops(mapmult);
        }
    }
}

Example 35 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class AggUnaryOp method optFindExecType.

@Override
protected ExecType optFindExecType() {
    checkAndSetForcedPlatform();
    ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
    // forced / memory-based / threshold-based decision
    if (_etypeForced != null) {
        _etype = _etypeForced;
    } else {
        if (OptimizerUtils.isMemoryBasedOptLevel()) {
            _etype = findExecTypeByMemEstimate();
        } else // Choose CP, if the input dimensions are below threshold or if the input is a vector
        if (getInput().get(0).areDimsBelowThreshold() || getInput().get(0).isVector()) {
            _etype = ExecType.CP;
        } else {
            _etype = REMOTE;
        }
        // check for valid CP dimensions and matrix size
        checkAndSetInvalidCPDimsAndSize();
    }
    // single parent also in spark because it's likely cheap and reduces data transfer)
    if (_etype == ExecType.CP && _etypeForced != ExecType.CP && // input is not checkpoint
    !(getInput().get(0) instanceof DataOp) && (// uagg is only parent, or
    getInput().get(0).getParent().size() == 1 || // w/o agg
    !requiresAggregation(getInput().get(0), _direction)) && getInput().get(0).optFindExecType() == ExecType.SPARK) {
        // pull unary aggregate into spark
        _etype = ExecType.SPARK;
    }
    // mark for recompile (forever)
    setRequiresRecompileIfNecessary();
    return _etype;
}