Examples with ExecType org.apache.sysml.lops.LopProperties.ExecType used on opensource projects

Example 21 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class DataGenOp method constructLops.

@Override
public Lop constructLops() throws HopsException, LopsException {
    //return already created lops
    if (getLops() != null)
        return getLops();
    ExecType et = optFindExecType();
    HashMap<String, Lop> inputLops = new HashMap<String, Lop>();
    for (Entry<String, Integer> cur : _paramIndexMap.entrySet()) {
        if (cur.getKey().equals(DataExpression.RAND_ROWS) && _dim1 > 0)
            inputLops.put(cur.getKey(), new LiteralOp(_dim1).constructLops());
        else if (cur.getKey().equals(DataExpression.RAND_COLS) && _dim2 > 0)
            inputLops.put(cur.getKey(), new LiteralOp(_dim2).constructLops());
        else
            inputLops.put(cur.getKey(), getInput().get(cur.getValue()).constructLops());
    }
    DataGen rnd = new DataGen(_op, _id, inputLops, _baseDir, getDataType(), getValueType(), et);
    int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads);
    rnd.setNumThreads(k);
    rnd.getOutputParameters().setDimensions(getDim1(), getDim2(), //robust handling for blocksize (important for -exec singlenode; otherwise incorrect results)
    (getRowsInBlock() > 0) ? getRowsInBlock() : ConfigurationManager.getBlocksize(), (getColsInBlock() > 0) ? getColsInBlock() : ConfigurationManager.getBlocksize(), //actual rand nnz might differ (in cp/mr they are corrected after execution)
    (_op == DataGenMethod.RAND && et == ExecType.SPARK && getNnz() != 0) ? -1 : getNnz(), getUpdateType());
    setLineNumbers(rnd);
    setLops(rnd);
    //add reblock/checkpoint lops if necessary
    constructAndSetLopsDataFlowProperties();
    return getLops();
}

Example 22 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class ConvolutionOp method optFindExecType.

@Override
protected ExecType optFindExecType() throws HopsException {
    checkAndSetForcedPlatform();
    ExecType REMOTE = OptimizerUtils.isSparkExecutionMode() ? ExecType.SPARK : ExecType.MR;
    if (_etypeForced != null) {
        _etype = findGPUExecTypeByMemEstimate(_etypeForced);
    } else {
        if (OptimizerUtils.isMemoryBasedOptLevel()) {
            _etype = findGPUExecTypeByMemEstimate(findExecTypeByMemEstimate());
        } else {
            _etype = REMOTE;
        }
        //check for valid CP dimensions and matrix size
        checkAndSetInvalidCPDimsAndSize();
    }
    // TODO: Fix this after adding remaining spark instructions
    _etype = !isEligibleForSpark() && _etype == REMOTE ? ExecType.CP : _etype;
    //mark for recompile (forever)
    if (ConfigurationManager.isDynamicRecompilation() && !dimsKnown(true) && _etype == REMOTE)
        setRequiresRecompile();
    return _etype;
}

Example 23 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class AggBinaryOp method constructCPLopsTSMM.

//////////////////////////
// CP Lops generation
/////////////////////////
private void constructCPLopsTSMM(MMTSJType mmtsj) throws HopsException, LopsException {
    int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads);
    ExecType et = ExecType.CP;
    if (DMLScript.USE_ACCELERATOR && (DMLScript.FORCE_ACCELERATOR || getMemEstimate() < OptimizerUtils.GPU_MEMORY_BUDGET)) {
        et = ExecType.GPU;
    }
    Lop matmultCP = new MMTSJ(getInput().get(mmtsj.isLeft() ? 1 : 0).constructLops(), getDataType(), getValueType(), et, mmtsj, false, k);
    matmultCP.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
    setLineNumbers(matmultCP);
    setLops(matmultCP);
}

Example 24 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class AggBinaryOp method constructLops.

/**
	 * NOTE: overestimated mem in case of transpose-identity matmult, but 3/2 at worst
	 *       and existing mem estimate advantageous in terms of consistency hops/lops,
	 *       and some special cases internally materialize the transpose for better cache locality  
	 */
@Override
public Lop constructLops() throws HopsException, LopsException {
    //return already created lops
    if (getLops() != null)
        return getLops();
    //construct matrix mult lops (currently only supported aggbinary)
    if (isMatrixMultiply()) {
        Hop input1 = getInput().get(0);
        Hop input2 = getInput().get(1);
        //matrix mult operation selection part 1 (CP vs MR vs Spark)
        ExecType et = optFindExecType();
        //matrix mult operation selection part 2 (specific pattern)
        //determine tsmm pattern
        MMTSJType mmtsj = checkTransposeSelf();
        //determine mmchain pattern
        ChainType chain = checkMapMultChain();
        if (et == ExecType.CP) {
            //matrix mult operation selection part 3 (CP type)
            _method = optFindMMultMethodCP(input1.getDim1(), input1.getDim2(), input2.getDim1(), input2.getDim2(), mmtsj, chain, _hasLeftPMInput);
            //dispatch CP lops construction 
            switch(_method) {
                case TSMM:
                    constructCPLopsTSMM(mmtsj);
                    break;
                case MAPMM_CHAIN:
                    constructCPLopsMMChain(chain);
                    break;
                case PMM:
                    constructCPLopsPMM();
                    break;
                case MM:
                    constructCPLopsMM();
                    break;
                default:
                    throw new HopsException(this.printErrorLocation() + "Invalid Matrix Mult Method (" + _method + ") while constructing CP lops.");
            }
        } else if (et == ExecType.SPARK) {
            //matrix mult operation selection part 3 (SPARK type)
            boolean tmmRewrite = HopRewriteUtils.isTransposeOperation(input1);
            _method = optFindMMultMethodSpark(input1.getDim1(), input1.getDim2(), input1.getRowsInBlock(), input1.getColsInBlock(), input1.getNnz(), input2.getDim1(), input2.getDim2(), input2.getRowsInBlock(), input2.getColsInBlock(), input2.getNnz(), mmtsj, chain, _hasLeftPMInput, tmmRewrite);
            //dispatch SPARK lops construction 
            switch(_method) {
                case TSMM:
                case TSMM2:
                    constructSparkLopsTSMM(mmtsj, _method == MMultMethod.TSMM2);
                    break;
                case MAPMM_L:
                case MAPMM_R:
                    constructSparkLopsMapMM(_method);
                    break;
                case MAPMM_CHAIN:
                    constructSparkLopsMapMMChain(chain);
                    break;
                case PMAPMM:
                    constructSparkLopsPMapMM();
                    break;
                case CPMM:
                    constructSparkLopsCPMM();
                    break;
                case RMM:
                    constructSparkLopsRMM();
                    break;
                case PMM:
                    constructSparkLopsPMM();
                    break;
                case ZIPMM:
                    constructSparkLopsZIPMM();
                    break;
                default:
                    throw new HopsException(this.printErrorLocation() + "Invalid Matrix Mult Method (" + _method + ") while constructing SPARK lops.");
            }
        } else if (et == ExecType.MR) {
            //matrix mult operation selection part 3 (MR type)
            _method = optFindMMultMethodMR(input1.getDim1(), input1.getDim2(), input1.getRowsInBlock(), input1.getColsInBlock(), input1.getNnz(), input2.getDim1(), input2.getDim2(), input2.getRowsInBlock(), input2.getColsInBlock(), input2.getNnz(), mmtsj, chain, _hasLeftPMInput);
            //dispatch MR lops construction
            switch(_method) {
                case MAPMM_L:
                case MAPMM_R:
                    constructMRLopsMapMM(_method);
                    break;
                case MAPMM_CHAIN:
                    constructMRLopsMapMMChain(chain);
                    break;
                case CPMM:
                    constructMRLopsCPMM();
                    break;
                case RMM:
                    constructMRLopsRMM();
                    break;
                case TSMM:
                    constructMRLopsTSMM(mmtsj);
                    break;
                case PMM:
                    constructMRLopsPMM();
                    break;
                default:
                    throw new HopsException(this.printErrorLocation() + "Invalid Matrix Mult Method (" + _method + ") while constructing MR lops.");
            }
        }
    } else
        throw new HopsException(this.printErrorLocation() + "Invalid operation in AggBinary Hop, aggBin(" + innerOp + "," + outerOp + ") while constructing lops.");
    //add reblock/checkpoint lops if necessary
    constructAndSetLopsDataFlowProperties();
    return getLops();
}

Example 25 with ExecType

use of org.apache.sysml.lops.LopProperties.ExecType in project incubator-systemml by apache.

the class AggBinaryOp method constructMRLopsPMM.

private void constructMRLopsPMM() throws HopsException, LopsException {
    //PMM has two potential modes (a) w/ full permutation matrix input, and 
    //(b) w/ already condensed input vector of target row positions.
    Hop pmInput = getInput().get(0);
    Hop rightInput = getInput().get(1);
    Lop lpmInput = pmInput.constructLops();
    Hop nrow = null;
    double mestPM = OptimizerUtils.estimateSize(pmInput.getDim1(), 1);
    ExecType etVect = (mestPM > OptimizerUtils.getLocalMemBudget()) ? ExecType.MR : ExecType.CP;
    //a) full permutation matrix input (potentially without empty block materialized)
    if (//not a vector
    pmInput.getDim2() != 1) {
        //compute condensed permutation matrix vector input			
        //v = rowMaxIndex(t(pm)) * rowMax(t(pm)) 
        ReorgOp transpose = HopRewriteUtils.createTranspose(pmInput);
        transpose.setForcedExecType(ExecType.MR);
        AggUnaryOp agg1 = HopRewriteUtils.createAggUnaryOp(transpose, AggOp.MAXINDEX, Direction.Row);
        agg1.setForcedExecType(ExecType.MR);
        AggUnaryOp agg2 = HopRewriteUtils.createAggUnaryOp(transpose, AggOp.MAX, Direction.Row);
        agg2.setForcedExecType(ExecType.MR);
        BinaryOp mult = HopRewriteUtils.createBinary(agg1, agg2, OpOp2.MULT);
        mult.setForcedExecType(ExecType.MR);
        //compute NROW target via nrow(m)
        nrow = HopRewriteUtils.createValueHop(pmInput, true);
        nrow.setOutputBlocksizes(0, 0);
        nrow.setForcedExecType(ExecType.CP);
        HopRewriteUtils.copyLineNumbers(this, nrow);
        lpmInput = mult.constructLops();
        HopRewriteUtils.removeChildReference(pmInput, transpose);
    } else //input vector
    {
        //compute NROW target via max(v)
        nrow = HopRewriteUtils.createAggUnaryOp(pmInput, AggOp.MAX, Direction.RowCol);
        nrow.setOutputBlocksizes(0, 0);
        nrow.setForcedExecType(etVect);
        HopRewriteUtils.copyLineNumbers(this, nrow);
    }
    //b) condensed permutation matrix vector input (target rows)		
    boolean needPart = !pmInput.dimsKnown() || pmInput.getDim1() > DistributedCacheInput.PARTITION_SIZE;
    if (needPart) {
        //requires partitioning
        lpmInput = new DataPartition(lpmInput, DataType.MATRIX, ValueType.DOUBLE, etVect, PDataPartitionFormat.ROW_BLOCK_WISE_N);
        lpmInput.getOutputParameters().setDimensions(pmInput.getDim1(), 1, getRowsInBlock(), getColsInBlock(), pmInput.getDim1());
        setLineNumbers(lpmInput);
    }
    _outputEmptyBlocks = !OptimizerUtils.allowsToFilterEmptyBlockOutputs(this);
    PMMJ pmm = new PMMJ(lpmInput, rightInput.constructLops(), nrow.constructLops(), getDataType(), getValueType(), needPart, _outputEmptyBlocks, ExecType.MR);
    pmm.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
    setLineNumbers(pmm);
    Aggregate aggregate = new Aggregate(pmm, HopsAgg2Lops.get(outerOp), getDataType(), getValueType(), ExecType.MR);
    aggregate.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
    // aggregation uses kahanSum but the inputs do not have correction values
    aggregate.setupCorrectionLocation(CorrectionLocationType.NONE);
    setLineNumbers(aggregate);
    setLops(aggregate);
    HopRewriteUtils.removeChildReference(pmInput, nrow);
}