Examples with PMMJ org.apache.sysml.lops.PMMJ used on opensource projects

Example 1 with PMMJ

use of org.apache.sysml.lops.PMMJ in project incubator-systemml by apache.

the class ParameterizedBuiltinOp method constructLopsRemoveEmpty.

private void constructLopsRemoveEmpty(HashMap<String, Lop> inputlops, ExecType et) throws HopsException, LopsException {
    Hop targetHop = getInput().get(_paramIndexMap.get("target"));
    Hop marginHop = getInput().get(_paramIndexMap.get("margin"));
    Hop selectHop = (_paramIndexMap.get("select") != null) ? getInput().get(_paramIndexMap.get("select")) : null;
    if (et == ExecType.CP || et == ExecType.CP_FILE) {
        ParameterizedBuiltin pbilop = new ParameterizedBuiltin(inputlops, HopsParameterizedBuiltinLops.get(_op), getDataType(), getValueType(), et);
        setOutputDimensions(pbilop);
        setLineNumbers(pbilop);
        setLops(pbilop);
    /*DISABLED CP PMM (see for example, MDA Bivar test, requires size propagation on recompile)
			if( et == ExecType.CP && isTargetDiagInput() && marginHop instanceof LiteralOp 
					 && ((LiteralOp)marginHop).getStringValue().equals("rows")
					 && _outputPermutationMatrix ) //SPECIAL CASE SELECTION VECTOR
			{				
				//TODO this special case could be taken into account for memory estimates in order
				// to reduce the estimates for the input diag and subsequent matrix multiply
				
				//get input vector (without materializing diag())
				Hop input = targetHop.getInput().get(0);
				long brlen = input.getRowsInBlock();
				long bclen = input.getColsInBlock();
				MemoTable memo = new MemoTable();
			
				boolean isPPredInput = (input instanceof BinaryOp && ((BinaryOp)input).isPPredOperation());
				
				//step1: compute index vectors
				Hop ppred0 = input;
				if( !isPPredInput ) { //ppred only if required
					ppred0 = new BinaryOp("tmp1", DataType.MATRIX, ValueType.DOUBLE, OpOp2.NOTEQUAL, input, new LiteralOp("0",0));
					HopRewriteUtils.setOutputBlocksizes(ppred0, brlen, bclen);
					ppred0.refreshSizeInformation();
					ppred0.computeMemEstimate(memo); //select exec type
					HopRewriteUtils.copyLineNumbers(this, ppred0);
				}
				
				UnaryOp cumsum = new UnaryOp("tmp2", DataType.MATRIX, ValueType.DOUBLE, OpOp1.CUMSUM, ppred0); 
				HopRewriteUtils.setOutputBlocksizes(cumsum, brlen, bclen);
				cumsum.refreshSizeInformation(); 
				cumsum.computeMemEstimate(memo); //select exec type
				HopRewriteUtils.copyLineNumbers(this, cumsum);	
			
				BinaryOp sel = new BinaryOp("tmp3", DataType.MATRIX, ValueType.DOUBLE, OpOp2.MULT, ppred0, cumsum);
				HopRewriteUtils.setOutputBlocksizes(sel, brlen, bclen); 
				sel.refreshSizeInformation();
				sel.computeMemEstimate(memo); //select exec type
				HopRewriteUtils.copyLineNumbers(this, sel);
				Lop loutput = sel.constructLops();
				
				//Step 4: cleanup hops (allow for garbage collection)
				HopRewriteUtils.removeChildReference(ppred0, input);
				
				setLops( loutput );
			}
			else //GENERAL CASE
			{
				ParameterizedBuiltin pbilop = new ParameterizedBuiltin( et, inputlops,
						HopsParameterizedBuiltinLops.get(_op), getDataType(), getValueType());
				
				pbilop.getOutputParameters().setDimensions(getDim1(),getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
				setLineNumbers(pbilop);
				setLops(pbilop);
			}
			*/
    } else if (et == ExecType.MR) {
        //special compile for mr removeEmpty-diag 
        if (isTargetDiagInput() && marginHop instanceof LiteralOp && ((LiteralOp) marginHop).getStringValue().equals("rows")) {
            //get input vector (without materializing diag())
            Hop input = targetHop.getInput().get(0);
            long brlen = input.getRowsInBlock();
            long bclen = input.getColsInBlock();
            MemoTable memo = new MemoTable();
            boolean isPPredInput = (input instanceof BinaryOp && ((BinaryOp) input).isPPredOperation());
            //step1: compute index vectors
            Hop ppred0 = input;
            if (!isPPredInput) {
                //ppred only if required
                ppred0 = HopRewriteUtils.createBinary(input, new LiteralOp(0), OpOp2.NOTEQUAL);
                HopRewriteUtils.updateHopCharacteristics(ppred0, brlen, bclen, memo, this);
            }
            UnaryOp cumsum = HopRewriteUtils.createUnary(ppred0, OpOp1.CUMSUM);
            HopRewriteUtils.updateHopCharacteristics(cumsum, brlen, bclen, memo, this);
            Lop loutput = null;
            double mest = AggBinaryOp.getMapmmMemEstimate(input.getDim1(), 1, brlen, bclen, -1, brlen, bclen, brlen, bclen, -1, 1, true);
            double mbudget = OptimizerUtils.getRemoteMemBudgetMap(true);
            if (//SPECIAL CASE: SELECTION VECTOR
            _outputPermutationMatrix && mest < mbudget) {
                BinaryOp sel = HopRewriteUtils.createBinary(ppred0, cumsum, OpOp2.MULT);
                HopRewriteUtils.updateHopCharacteristics(sel, brlen, bclen, memo, this);
                loutput = sel.constructLops();
            } else //GENERAL CASE: GENERAL PERMUTATION MATRIX
            {
                //max ensures non-zero entries and at least one output row
                BinaryOp max = HopRewriteUtils.createBinary(cumsum, new LiteralOp(1), OpOp2.MAX);
                HopRewriteUtils.updateHopCharacteristics(max, brlen, bclen, memo, this);
                DataGenOp seq = HopRewriteUtils.createSeqDataGenOp(input);
                seq.setName("tmp4");
                HopRewriteUtils.updateHopCharacteristics(seq, brlen, bclen, memo, this);
                //step 2: compute removeEmpty(rows) output via table, seq guarantees right column dimension
                //note: weights always the input (even if isPPredInput) because input also includes 0s
                TernaryOp table = new TernaryOp("tmp5", DataType.MATRIX, ValueType.DOUBLE, OpOp3.CTABLE, max, seq, input);
                table.setOutputBlocksizes(brlen, bclen);
                table.refreshSizeInformation();
                //force MR 
                table.setForcedExecType(ExecType.MR);
                HopRewriteUtils.copyLineNumbers(this, table);
                table.setDisjointInputs(true);
                table.setOutputEmptyBlocks(_outputEmptyBlocks);
                loutput = table.constructLops();
                HopRewriteUtils.removeChildReference(table, input);
            }
            //Step 4: cleanup hops (allow for garbage collection)
            HopRewriteUtils.removeChildReference(ppred0, input);
            setLops(loutput);
        } else //default mr remove empty
        if (et == ExecType.MR) {
            if (!(marginHop instanceof LiteralOp))
                throw new HopsException("Parameter 'margin' must be a literal argument.");
            Hop input = targetHop;
            long rlen = input.getDim1();
            long clen = input.getDim2();
            long brlen = input.getRowsInBlock();
            long bclen = input.getColsInBlock();
            long nnz = input.getNnz();
            boolean rmRows = ((LiteralOp) marginHop).getStringValue().equals("rows");
            //construct lops via new partial hop dag and subsequent lops construction 
            //in order to reuse of operator selection decisions
            BinaryOp ppred0 = null;
            Hop emptyInd = null;
            if (selectHop == null) {
                //Step1: compute row/col non-empty indicators 
                ppred0 = HopRewriteUtils.createBinary(input, new LiteralOp(0), OpOp2.NOTEQUAL);
                //always MR 
                ppred0.setForcedExecType(ExecType.MR);
                emptyInd = ppred0;
                if (!((rmRows && clen == 1) || (!rmRows && rlen == 1))) {
                    emptyInd = HopRewriteUtils.createAggUnaryOp(ppred0, AggOp.MAX, rmRows ? Direction.Row : Direction.Col);
                    //always MR
                    emptyInd.setForcedExecType(ExecType.MR);
                    HopRewriteUtils.copyLineNumbers(this, emptyInd);
                }
            } else {
                emptyInd = selectHop;
                emptyInd.setOutputBlocksizes(brlen, bclen);
                emptyInd.refreshSizeInformation();
                //always MR
                emptyInd.setForcedExecType(ExecType.MR);
                HopRewriteUtils.copyLineNumbers(this, emptyInd);
            }
            //Step 2: compute row offsets for non-empty rows
            Hop cumsumInput = emptyInd;
            if (!rmRows) {
                cumsumInput = HopRewriteUtils.createTranspose(emptyInd);
                HopRewriteUtils.updateHopCharacteristics(cumsumInput, brlen, bclen, this);
            }
            UnaryOp cumsum = HopRewriteUtils.createUnary(cumsumInput, OpOp1.CUMSUM);
            HopRewriteUtils.updateHopCharacteristics(cumsum, brlen, bclen, this);
            Hop cumsumOutput = cumsum;
            if (!rmRows) {
                cumsumOutput = HopRewriteUtils.createTranspose(cumsum);
                HopRewriteUtils.updateHopCharacteristics(cumsumOutput, brlen, bclen, this);
            }
            //alternative: right indexing
            Hop maxDim = HopRewriteUtils.createAggUnaryOp(cumsumOutput, AggOp.MAX, Direction.RowCol);
            HopRewriteUtils.updateHopCharacteristics(maxDim, brlen, bclen, this);
            BinaryOp offsets = HopRewriteUtils.createBinary(cumsumOutput, emptyInd, OpOp2.MULT);
            HopRewriteUtils.updateHopCharacteristics(offsets, brlen, bclen, this);
            //Step 3: gather non-empty rows/cols into final results 
            Lop linput = input.constructLops();
            Lop loffset = offsets.constructLops();
            Lop lmaxdim = maxDim.constructLops();
            double mestPM = OptimizerUtils.estimatePartitionedSizeExactSparsity(rlen, 1, brlen, bclen, 1.0);
            Lop rmEmpty = null;
            //a) broadcast-based PMM (permutation matrix mult)
            if (rmRows && rlen > 0 && mestPM < OptimizerUtils.getRemoteMemBudgetMap()) {
                boolean needPart = !offsets.dimsKnown() || offsets.getDim1() > DistributedCacheInput.PARTITION_SIZE;
                if (needPart) {
                    //requires partitioning
                    loffset = new DataPartition(loffset, DataType.MATRIX, ValueType.DOUBLE, (mestPM > OptimizerUtils.getLocalMemBudget()) ? ExecType.MR : ExecType.CP, PDataPartitionFormat.ROW_BLOCK_WISE_N);
                    loffset.getOutputParameters().setDimensions(rlen, 1, brlen, bclen, rlen);
                    setLineNumbers(loffset);
                }
                rmEmpty = new PMMJ(loffset, linput, lmaxdim, getDataType(), getValueType(), needPart, true, ExecType.MR);
                setOutputDimensions(rmEmpty);
                setLineNumbers(rmEmpty);
            } else //b) general case: repartition-based rmempty
            {
                boolean requiresRep = ((clen > bclen || clen <= 0) && rmRows) || ((rlen > brlen || rlen <= 0) && !rmRows);
                if (requiresRep) {
                    //ncol of left input (determines num replicates)
                    Lop pos = createOffsetLop(input, rmRows);
                    loffset = new RepMat(loffset, pos, rmRows, DataType.MATRIX, ValueType.DOUBLE);
                    loffset.getOutputParameters().setDimensions(rlen, clen, brlen, bclen, nnz);
                    setLineNumbers(loffset);
                }
                Group group1 = new Group(linput, Group.OperationTypes.Sort, getDataType(), getValueType());
                setLineNumbers(group1);
                group1.getOutputParameters().setDimensions(rlen, clen, brlen, bclen, nnz);
                Group group2 = new Group(loffset, Group.OperationTypes.Sort, getDataType(), getValueType());
                setLineNumbers(group2);
                group2.getOutputParameters().setDimensions(rlen, clen, brlen, bclen, nnz);
                HashMap<String, Lop> inMap = new HashMap<String, Lop>();
                inMap.put("target", group1);
                inMap.put("offset", group2);
                inMap.put("maxdim", lmaxdim);
                inMap.put("margin", inputlops.get("margin"));
                rmEmpty = new ParameterizedBuiltin(inMap, HopsParameterizedBuiltinLops.get(_op), getDataType(), getValueType(), et);
                setOutputDimensions(rmEmpty);
                setLineNumbers(rmEmpty);
            }
            Group group3 = new Group(rmEmpty, Group.OperationTypes.Sort, getDataType(), getValueType());
            setLineNumbers(group3);
            group3.getOutputParameters().setDimensions(-1, -1, brlen, bclen, -1);
            Aggregate finalagg = new Aggregate(group3, Aggregate.OperationTypes.Sum, DataType.MATRIX, getValueType(), ExecType.MR);
            setOutputDimensions(finalagg);
            setLineNumbers(finalagg);
            //Step 4: cleanup hops (allow for garbage collection)
            if (selectHop == null)
                HopRewriteUtils.removeChildReference(ppred0, input);
            setLops(finalagg);
        }
    } else if (et == ExecType.SPARK) {
        if (!(marginHop instanceof LiteralOp))
            throw new HopsException("Parameter 'margin' must be a literal argument.");
        Hop input = targetHop;
        long rlen = input.getDim1();
        long clen = input.getDim2();
        long brlen = input.getRowsInBlock();
        long bclen = input.getColsInBlock();
        boolean rmRows = ((LiteralOp) marginHop).getStringValue().equals("rows");
        //construct lops via new partial hop dag and subsequent lops construction 
        //in order to reuse of operator selection decisions
        BinaryOp ppred0 = null;
        Hop emptyInd = null;
        if (selectHop == null) {
            //Step1: compute row/col non-empty indicators 
            ppred0 = HopRewriteUtils.createBinary(input, new LiteralOp(0), OpOp2.NOTEQUAL);
            //always Spark
            ppred0.setForcedExecType(ExecType.SPARK);
            emptyInd = ppred0;
            if (!((rmRows && clen == 1) || (!rmRows && rlen == 1))) {
                emptyInd = HopRewriteUtils.createAggUnaryOp(ppred0, AggOp.MAX, rmRows ? Direction.Row : Direction.Col);
                //always Spark
                emptyInd.setForcedExecType(ExecType.SPARK);
            }
        } else {
            emptyInd = selectHop;
            emptyInd.setOutputBlocksizes(brlen, bclen);
            emptyInd.refreshSizeInformation();
            //always Spark
            emptyInd.setForcedExecType(ExecType.SPARK);
            HopRewriteUtils.copyLineNumbers(this, emptyInd);
        }
        //Step 2: compute row offsets for non-empty rows
        Hop cumsumInput = emptyInd;
        if (!rmRows) {
            cumsumInput = HopRewriteUtils.createTranspose(emptyInd);
            HopRewriteUtils.updateHopCharacteristics(cumsumInput, brlen, bclen, this);
        }
        UnaryOp cumsum = HopRewriteUtils.createUnary(cumsumInput, OpOp1.CUMSUM);
        HopRewriteUtils.updateHopCharacteristics(cumsum, brlen, bclen, this);
        Hop cumsumOutput = cumsum;
        if (!rmRows) {
            cumsumOutput = HopRewriteUtils.createTranspose(cumsum);
            HopRewriteUtils.updateHopCharacteristics(cumsumOutput, brlen, bclen, this);
        }
        //alternative: right indexing
        Hop maxDim = HopRewriteUtils.createAggUnaryOp(cumsumOutput, AggOp.MAX, Direction.RowCol);
        HopRewriteUtils.updateHopCharacteristics(maxDim, brlen, bclen, this);
        BinaryOp offsets = HopRewriteUtils.createBinary(cumsumOutput, emptyInd, OpOp2.MULT);
        HopRewriteUtils.updateHopCharacteristics(offsets, brlen, bclen, this);
        //Step 3: gather non-empty rows/cols into final results 
        Lop linput = input.constructLops();
        Lop loffset = offsets.constructLops();
        Lop lmaxdim = maxDim.constructLops();
        HashMap<String, Lop> inMap = new HashMap<String, Lop>();
        inMap.put("target", linput);
        inMap.put("offset", loffset);
        inMap.put("maxdim", lmaxdim);
        inMap.put("margin", inputlops.get("margin"));
        if (!FORCE_DIST_RM_EMPTY && isRemoveEmptyBcSP())
            _bRmEmptyBC = true;
        ParameterizedBuiltin pbilop = new ParameterizedBuiltin(inMap, HopsParameterizedBuiltinLops.get(_op), getDataType(), getValueType(), et, _bRmEmptyBC);
        setOutputDimensions(pbilop);
        setLineNumbers(pbilop);
        //Step 4: cleanup hops (allow for garbage collection)
        if (selectHop == null)
            HopRewriteUtils.removeChildReference(ppred0, input);
        setLops(pbilop);
    //NOTE: in contrast to mr, replication and aggregation handled instruction-local
    }
}

Example 2 with PMMJ

use of org.apache.sysml.lops.PMMJ in project incubator-systemml by apache.

the class AggBinaryOp method constructSparkLopsPMM.

private void constructSparkLopsPMM() throws HopsException, LopsException {
    //PMM has two potential modes (a) w/ full permutation matrix input, and 
    //(b) w/ already condensed input vector of target row positions.
    Hop pmInput = getInput().get(0);
    Hop rightInput = getInput().get(1);
    Lop lpmInput = pmInput.constructLops();
    Hop nrow = null;
    double mestPM = OptimizerUtils.estimateSize(pmInput.getDim1(), 1);
    ExecType etVect = (mestPM > OptimizerUtils.getLocalMemBudget()) ? ExecType.MR : ExecType.CP;
    //a) full permutation matrix input (potentially without empty block materialized)
    if (//not a vector
    pmInput.getDim2() != 1) {
        //compute condensed permutation matrix vector input			
        //v = rowMaxIndex(t(pm)) * rowMax(t(pm)) 
        ReorgOp transpose = HopRewriteUtils.createTranspose(pmInput);
        transpose.setForcedExecType(ExecType.SPARK);
        AggUnaryOp agg1 = HopRewriteUtils.createAggUnaryOp(transpose, AggOp.MAXINDEX, Direction.Row);
        agg1.setForcedExecType(ExecType.SPARK);
        AggUnaryOp agg2 = HopRewriteUtils.createAggUnaryOp(transpose, AggOp.MAX, Direction.Row);
        agg2.setForcedExecType(ExecType.SPARK);
        BinaryOp mult = HopRewriteUtils.createBinary(agg1, agg2, OpOp2.MULT);
        mult.setForcedExecType(ExecType.SPARK);
        //compute NROW target via nrow(m)
        nrow = HopRewriteUtils.createValueHop(pmInput, true);
        nrow.setOutputBlocksizes(0, 0);
        nrow.setForcedExecType(ExecType.CP);
        HopRewriteUtils.copyLineNumbers(this, nrow);
        lpmInput = mult.constructLops();
        HopRewriteUtils.removeChildReference(pmInput, transpose);
    } else //input vector
    {
        //compute NROW target via max(v)
        nrow = HopRewriteUtils.createAggUnaryOp(pmInput, AggOp.MAX, Direction.RowCol);
        nrow.setOutputBlocksizes(0, 0);
        nrow.setForcedExecType(etVect);
        HopRewriteUtils.copyLineNumbers(this, nrow);
    }
    //b) condensed permutation matrix vector input (target rows)
    _outputEmptyBlocks = !OptimizerUtils.allowsToFilterEmptyBlockOutputs(this);
    PMMJ pmm = new PMMJ(lpmInput, rightInput.constructLops(), nrow.constructLops(), getDataType(), getValueType(), false, _outputEmptyBlocks, ExecType.SPARK);
    setOutputDimensions(pmm);
    setLineNumbers(pmm);
    setLops(pmm);
    HopRewriteUtils.removeChildReference(pmInput, nrow);
}

Example 3 with PMMJ

use of org.apache.sysml.lops.PMMJ in project incubator-systemml by apache.

the class AggBinaryOp method constructMRLopsPMM.

private void constructMRLopsPMM() throws HopsException, LopsException {
    //PMM has two potential modes (a) w/ full permutation matrix input, and 
    //(b) w/ already condensed input vector of target row positions.
    Hop pmInput = getInput().get(0);
    Hop rightInput = getInput().get(1);
    Lop lpmInput = pmInput.constructLops();
    Hop nrow = null;
    double mestPM = OptimizerUtils.estimateSize(pmInput.getDim1(), 1);
    ExecType etVect = (mestPM > OptimizerUtils.getLocalMemBudget()) ? ExecType.MR : ExecType.CP;
    //a) full permutation matrix input (potentially without empty block materialized)
    if (//not a vector
    pmInput.getDim2() != 1) {
        //compute condensed permutation matrix vector input			
        //v = rowMaxIndex(t(pm)) * rowMax(t(pm)) 
        ReorgOp transpose = HopRewriteUtils.createTranspose(pmInput);
        transpose.setForcedExecType(ExecType.MR);
        AggUnaryOp agg1 = HopRewriteUtils.createAggUnaryOp(transpose, AggOp.MAXINDEX, Direction.Row);
        agg1.setForcedExecType(ExecType.MR);
        AggUnaryOp agg2 = HopRewriteUtils.createAggUnaryOp(transpose, AggOp.MAX, Direction.Row);
        agg2.setForcedExecType(ExecType.MR);
        BinaryOp mult = HopRewriteUtils.createBinary(agg1, agg2, OpOp2.MULT);
        mult.setForcedExecType(ExecType.MR);
        //compute NROW target via nrow(m)
        nrow = HopRewriteUtils.createValueHop(pmInput, true);
        nrow.setOutputBlocksizes(0, 0);
        nrow.setForcedExecType(ExecType.CP);
        HopRewriteUtils.copyLineNumbers(this, nrow);
        lpmInput = mult.constructLops();
        HopRewriteUtils.removeChildReference(pmInput, transpose);
    } else //input vector
    {
        //compute NROW target via max(v)
        nrow = HopRewriteUtils.createAggUnaryOp(pmInput, AggOp.MAX, Direction.RowCol);
        nrow.setOutputBlocksizes(0, 0);
        nrow.setForcedExecType(etVect);
        HopRewriteUtils.copyLineNumbers(this, nrow);
    }
    //b) condensed permutation matrix vector input (target rows)		
    boolean needPart = !pmInput.dimsKnown() || pmInput.getDim1() > DistributedCacheInput.PARTITION_SIZE;
    if (needPart) {
        //requires partitioning
        lpmInput = new DataPartition(lpmInput, DataType.MATRIX, ValueType.DOUBLE, etVect, PDataPartitionFormat.ROW_BLOCK_WISE_N);
        lpmInput.getOutputParameters().setDimensions(pmInput.getDim1(), 1, getRowsInBlock(), getColsInBlock(), pmInput.getDim1());
        setLineNumbers(lpmInput);
    }
    _outputEmptyBlocks = !OptimizerUtils.allowsToFilterEmptyBlockOutputs(this);
    PMMJ pmm = new PMMJ(lpmInput, rightInput.constructLops(), nrow.constructLops(), getDataType(), getValueType(), needPart, _outputEmptyBlocks, ExecType.MR);
    pmm.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
    setLineNumbers(pmm);
    Aggregate aggregate = new Aggregate(pmm, HopsAgg2Lops.get(outerOp), getDataType(), getValueType(), ExecType.MR);
    aggregate.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
    // aggregation uses kahanSum but the inputs do not have correction values
    aggregate.setupCorrectionLocation(CorrectionLocationType.NONE);
    setLineNumbers(aggregate);
    setLops(aggregate);
    HopRewriteUtils.removeChildReference(pmInput, nrow);
}

Example 4 with PMMJ

use of org.apache.sysml.lops.PMMJ in project incubator-systemml by apache.

the class AggBinaryOp method constructCPLopsPMM.

/**
	 * NOTE: exists for consistency since removeEmtpy might be scheduled to MR
	 * but matrix mult on small output might be scheduled to CP. Hence, we 
	 * need to handle directly passed selection vectors in CP as well.
	 * 
	 * @throws HopsException if HopsException occurs
	 * @throws LopsException if LopsException occurs
	 */
private void constructCPLopsPMM() throws HopsException, LopsException {
    Hop pmInput = getInput().get(0);
    Hop rightInput = getInput().get(1);
    //NROW
    Hop nrow = HopRewriteUtils.createValueHop(pmInput, true);
    nrow.setOutputBlocksizes(0, 0);
    nrow.setForcedExecType(ExecType.CP);
    HopRewriteUtils.copyLineNumbers(this, nrow);
    Lop lnrow = nrow.constructLops();
    PMMJ pmm = new PMMJ(pmInput.constructLops(), rightInput.constructLops(), lnrow, getDataType(), getValueType(), false, false, ExecType.CP);
    //set degree of parallelism
    int k = OptimizerUtils.getConstrainedNumThreads(_maxNumThreads);
    pmm.setNumThreads(k);
    pmm.getOutputParameters().setDimensions(getDim1(), getDim2(), getRowsInBlock(), getColsInBlock(), getNnz());
    setLineNumbers(pmm);
    setLops(pmm);
    HopRewriteUtils.removeChildReference(pmInput, nrow);
}

Example 5 with PMMJ

use of org.apache.sysml.lops.PMMJ in project incubator-systemml by apache.

the class Dag method computeFootprintInMapper.

/**
	 * Computes the memory footprint required to execute <code>node</code> in the mapper.
	 * It is used only for those nodes that use inputs from distributed cache. The returned 
	 * value is utilized in limiting the number of instructions piggybacked onto a single GMR mapper.
	 * 
	 * @param node low-level operator
	 * @return memory footprint
	 */
private static double computeFootprintInMapper(Lop node) {
    // Memory limits must be checked only for nodes that use distributed cache
    if (!node.usesDistributedCache())
        // default behavior
        return 0.0;
    OutputParameters in1dims = node.getInputs().get(0).getOutputParameters();
    OutputParameters in2dims = node.getInputs().get(1).getOutputParameters();
    double footprint = 0;
    if (node instanceof MapMult) {
        int dcInputIndex = node.distributedCacheInputIndex()[0];
        footprint = AggBinaryOp.getMapmmMemEstimate(in1dims.getNumRows(), in1dims.getNumCols(), in1dims.getRowsInBlock(), in1dims.getColsInBlock(), in1dims.getNnz(), in2dims.getNumRows(), in2dims.getNumCols(), in2dims.getRowsInBlock(), in2dims.getColsInBlock(), in2dims.getNnz(), dcInputIndex, false);
    } else if (node instanceof PMMJ) {
        int dcInputIndex = node.distributedCacheInputIndex()[0];
        footprint = AggBinaryOp.getMapmmMemEstimate(in1dims.getNumRows(), 1, in1dims.getRowsInBlock(), in1dims.getColsInBlock(), in1dims.getNnz(), in2dims.getNumRows(), in2dims.getNumCols(), in2dims.getRowsInBlock(), in2dims.getColsInBlock(), in2dims.getNnz(), dcInputIndex, true);
    } else if (node instanceof AppendM) {
        footprint = BinaryOp.footprintInMapper(in1dims.getNumRows(), in1dims.getNumCols(), in2dims.getNumRows(), in2dims.getNumCols(), in1dims.getRowsInBlock(), in1dims.getColsInBlock());
    } else if (node instanceof BinaryM) {
        footprint = BinaryOp.footprintInMapper(in1dims.getNumRows(), in1dims.getNumCols(), in2dims.getNumRows(), in2dims.getNumCols(), in1dims.getRowsInBlock(), in1dims.getColsInBlock());
    } else {
        // default behavior
        return 0.0;
    }
    return footprint;
}