Examples with ReduceCol org.apache.sysml.runtime.functionobjects.ReduceCol used on opensource projects

Example 6 with ReduceCol

use of org.apache.sysml.runtime.functionobjects.ReduceCol in project incubator-systemml by apache.

the class UaggOuterChainCPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) throws DMLRuntimeException {
    boolean rightCached = (_uaggOp.indexFn instanceof ReduceCol || _uaggOp.indexFn instanceof ReduceAll || !LibMatrixOuterAgg.isSupportedUaggOp(_uaggOp, _bOp));
    MatrixBlock mbLeft = null, mbRight = null, mbOut = null;
    //get the main data input
    if (rightCached) {
        mbLeft = ec.getMatrixInput(input1.getName());
        mbRight = ec.getMatrixInput(input2.getName());
    } else {
        mbLeft = ec.getMatrixInput(input2.getName());
        mbRight = ec.getMatrixInput(input1.getName());
    }
    mbOut = mbLeft.uaggouterchainOperations(mbLeft, mbRight, mbOut, _bOp, _uaggOp);
    //release locks
    ec.releaseMatrixInput(input1.getName());
    ec.releaseMatrixInput(input2.getName());
    if (_uaggOp.aggOp.correctionExists)
        mbOut.dropLastRowsOrColums(_uaggOp.aggOp.correctionLocation);
    String output_name = output.getName();
    //final aggregation if required
    if (//RC AGG (output is scalar)
    _uaggOp.indexFn instanceof ReduceAll) {
        //create and set output scalar
        ScalarObject ret = null;
        switch(output.getValueType()) {
            case DOUBLE:
                ret = new DoubleObject(output_name, mbOut.quickGetValue(0, 0));
                break;
            default:
                throw new DMLRuntimeException("Invalid output value type: " + output.getValueType());
        }
        ec.setScalarOutput(output_name, ret);
    } else //R/C AGG (output is rdd)
    {
        //Additional memory requirement to convert from dense to sparse can be leveraged from released memory needed for input data above.
        mbOut.examSparsity();
        ec.setMatrixOutput(output_name, mbOut);
    }
}

Example 7 with ReduceCol

use of org.apache.sysml.runtime.functionobjects.ReduceCol in project incubator-systemml by apache.

the class UaggOuterChainSPInstruction method updateUnaryAggOutputMatrixCharacteristics.

protected void updateUnaryAggOutputMatrixCharacteristics(SparkExecutionContext sec) throws DMLRuntimeException {
    String strInput1Name, strInput2Name;
    if (_uaggOp.indexFn instanceof ReduceCol) {
        strInput1Name = input1.getName();
        strInput2Name = input2.getName();
    } else {
        strInput1Name = input2.getName();
        strInput2Name = input1.getName();
    }
    MatrixCharacteristics mc1 = sec.getMatrixCharacteristics(strInput1Name);
    MatrixCharacteristics mc2 = sec.getMatrixCharacteristics(strInput2Name);
    MatrixCharacteristics mcOut = sec.getMatrixCharacteristics(output.getName());
    if (!mcOut.dimsKnown()) {
        if (!mc1.dimsKnown()) {
            throw new DMLRuntimeException("The output dimensions are not specified and cannot be inferred from input:" + mc1.toString() + " " + mcOut.toString());
        } else {
            //infer statistics from input based on operator
            if (_uaggOp.indexFn instanceof ReduceAll)
                mcOut.set(1, 1, mc1.getRowsPerBlock(), mc1.getColsPerBlock());
            else if (_uaggOp.indexFn instanceof ReduceCol)
                mcOut.set(mc1.getRows(), 1, mc1.getRowsPerBlock(), mc1.getColsPerBlock());
            else if (_uaggOp.indexFn instanceof ReduceRow)
                mcOut.set(1, mc2.getCols(), mc1.getRowsPerBlock(), mc2.getColsPerBlock());
        }
    }
}

Example 8 with ReduceCol

use of org.apache.sysml.runtime.functionobjects.ReduceCol in project incubator-systemml by apache.

the class UaggOuterChainSPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) throws DMLRuntimeException {
    SparkExecutionContext sec = (SparkExecutionContext) ec;
    boolean rightCached = (_uaggOp.indexFn instanceof ReduceCol || _uaggOp.indexFn instanceof ReduceAll || !LibMatrixOuterAgg.isSupportedUaggOp(_uaggOp, _bOp));
    String rddVar = (rightCached) ? input1.getName() : input2.getName();
    String bcastVar = (rightCached) ? input2.getName() : input1.getName();
    //get rdd input
    JavaPairRDD<MatrixIndexes, MatrixBlock> in1 = sec.getBinaryBlockRDDHandleForVariable(rddVar);
    MatrixCharacteristics mcIn = sec.getMatrixCharacteristics(rddVar);
    boolean noKeyChange = preservesPartitioning(mcIn, _uaggOp.indexFn);
    //execute UAggOuterChain instruction
    JavaPairRDD<MatrixIndexes, MatrixBlock> out = null;
    if (LibMatrixOuterAgg.isSupportedUaggOp(_uaggOp, _bOp)) {
        //create sorted broadcast matrix 
        MatrixBlock mb = sec.getMatrixInput(bcastVar);
        sec.releaseMatrixInput(bcastVar);
        //prevent lineage tracking
        bcastVar = null;
        double[] vmb = DataConverter.convertToDoubleVector(mb);
        Broadcast<int[]> bvi = null;
        if (_uaggOp.aggOp.increOp.fn instanceof Builtin) {
            int[] vix = LibMatrixOuterAgg.prepareRowIndices(mb.getNumColumns(), vmb, _bOp, _uaggOp);
            bvi = sec.getSparkContext().broadcast(vix);
        } else
            Arrays.sort(vmb);
        Broadcast<double[]> bv = sec.getSparkContext().broadcast(vmb);
        //partitioning-preserving map-to-pair (under constraints)
        out = in1.mapPartitionsToPair(new RDDMapUAggOuterChainFunction(bv, bvi, _bOp, _uaggOp), noKeyChange);
    } else {
        PartitionedBroadcast<MatrixBlock> bv = sec.getBroadcastForVariable(bcastVar);
        //partitioning-preserving map-to-pair (under constraints)
        out = in1.mapPartitionsToPair(new RDDMapGenUAggOuterChainFunction(bv, _uaggOp, _aggOp, _bOp, mcIn), noKeyChange);
    }
    //final aggregation if required
    if (//RC AGG (output is scalar)
    _uaggOp.indexFn instanceof ReduceAll) {
        MatrixBlock tmp = RDDAggregateUtils.aggStable(out, _aggOp);
        //drop correction after aggregation
        tmp.dropLastRowsOrColums(_aggOp.correctionLocation);
        //put output block into symbol table (no lineage because single block)
        sec.setMatrixOutput(output.getName(), tmp);
    } else //R/C AGG (output is rdd)
    {
        //put output RDD handle into symbol table
        updateUnaryAggOutputMatrixCharacteristics(sec);
        if (_uaggOp.aggOp.correctionExists)
            out = out.mapValues(new AggregateDropCorrectionFunction(_uaggOp.aggOp));
        sec.setRDDHandleForVariable(output.getName(), out);
        sec.addLineageRDD(output.getName(), rddVar);
        if (bcastVar != null)
            sec.addLineageBroadcast(output.getName(), bcastVar);
    }
}

Example 9 with ReduceCol

use of org.apache.sysml.runtime.functionobjects.ReduceCol in project incubator-systemml by apache.

the class AggregateUnaryGPUInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) throws DMLRuntimeException {
    GPUStatistics.incrementNoOfExecutedGPUInst();
    String opcode = getOpcode();
    // nrow, ncol & length should either read or refresh metadata
    if (opcode.equalsIgnoreCase("nrow") || opcode.equalsIgnoreCase("ncol") || opcode.equalsIgnoreCase("length")) {
        throw new DMLRuntimeException("nrow, ncol & length should not be compiled as GPU instructions!");
    }
    //get inputs
    MatrixObject in1 = getMatrixInputForGPUInstruction(ec, _input1.getName());
    int rlen = (int) in1.getNumRows();
    int clen = (int) in1.getNumColumns();
    IndexFunction indexFunction = ((AggregateUnaryOperator) _optr).indexFn;
    if (indexFunction instanceof ReduceRow) {
        // COL{SUM, MAX...}
        ec.setMetaData(_output.getName(), 1, clen);
    } else if (indexFunction instanceof ReduceCol) {
        // ROW{SUM, MAX,...}
        ec.setMetaData(_output.getName(), rlen, 1);
    }
    LibMatrixCUDA.unaryAggregate(ec, ec.getGPUContext(), getExtendedOpcode(), in1, _output.getName(), (AggregateUnaryOperator) _optr);
    //release inputs/outputs
    ec.releaseMatrixInputForGPUInstruction(_input1.getName());
    // and set in the execution context by invoking the setScalarOutput
    if (indexFunction instanceof ReduceRow || indexFunction instanceof ReduceCol) {
        ec.releaseMatrixOutputForGPUInstruction(_output.getName());
    }
}

Example 10 with ReduceCol

use of org.apache.sysml.runtime.functionobjects.ReduceCol in project incubator-systemml by apache.

the class UaggOuterChainInstruction method processInstruction.

@Override
public void processInstruction(Class<? extends MatrixValue> valueClass, CachedValueMap cachedValues, IndexedMatrixValue tempValue, IndexedMatrixValue zeroInput, int blockRowFactor, int blockColFactor) throws DMLRuntimeException {
    ArrayList<IndexedMatrixValue> blkList = null;
    boolean rightCached = (_uaggOp.indexFn instanceof ReduceCol || _uaggOp.indexFn instanceof ReduceAll || !LibMatrixOuterAgg.isSupportedUaggOp(_uaggOp, _bOp));
    //get the main data input
    if (rightCached)
        blkList = cachedValues.get(input1);
    else
        // ReduceRow
        blkList = cachedValues.get(input2);
    if (blkList == null)
        return;
    for (IndexedMatrixValue imv : blkList) {
        if (imv == null)
            continue;
        MatrixIndexes in1Ix = imv.getIndexes();
        MatrixValue in1Val = imv.getValue();
        //allocate space for the intermediate and output value
        IndexedMatrixValue iout = cachedValues.holdPlace(output, valueClass);
        MatrixIndexes outIx = iout.getIndexes();
        MatrixValue outVal = iout.getValue();
        MatrixBlock corr = null;
        //get the distributed cache input
        byte dcInputIx = rightCached ? input2 : input1;
        DistributedCacheInput dcInput = MRBaseForCommonInstructions.dcValues.get(dcInputIx);
        //process instruction
        if (LibMatrixOuterAgg.isSupportedUaggOp(_uaggOp, _bOp)) {
            if ((LibMatrixOuterAgg.isRowIndexMax(_uaggOp)) || (LibMatrixOuterAgg.isRowIndexMin(_uaggOp))) {
                if (_bv == null) {
                    if (rightCached)
                        _bv = dcInput.getRowVectorArray();
                    else
                        _bv = dcInput.getColumnVectorArray();
                    _bvi = LibMatrixOuterAgg.prepareRowIndices(_bv.length, _bv, _bOp, _uaggOp);
                }
            } else {
                //step 1: prepare sorted rhs input (once per task)
                if (_bv == null) {
                    if (rightCached)
                        _bv = dcInput.getRowVectorArray();
                    else
                        _bv = dcInput.getColumnVectorArray();
                    Arrays.sort(_bv);
                }
            }
            LibMatrixOuterAgg.resetOutputMatix(in1Ix, (MatrixBlock) in1Val, outIx, (MatrixBlock) outVal, _uaggOp);
            LibMatrixOuterAgg.aggregateMatrix((MatrixBlock) in1Val, (MatrixBlock) outVal, _bv, _bvi, _bOp, _uaggOp);
        } else //default case 
        {
            long in2_cols = dcInput.getNumCols();
            long in2_colBlocks = (long) Math.ceil(((double) in2_cols) / dcInput.getNumColsPerBlock());
            for (int bidx = 1; bidx <= in2_colBlocks; bidx++) {
                IndexedMatrixValue imv2 = dcInput.getDataBlock(1, bidx);
                MatrixValue in2Val = imv2.getValue();
                //outer block operation
                OperationsOnMatrixValues.performBinaryIgnoreIndexes(in1Val, in2Val, _tmpVal1, _bOp);
                //unary aggregate operation
                OperationsOnMatrixValues.performAggregateUnary(in1Ix, _tmpVal1, outIx, _tmpVal2, _uaggOp, blockRowFactor, blockColFactor);
                //aggregate over all rhs blocks
                if (corr == null) {
                    outVal.reset(_tmpVal2.getNumRows(), _tmpVal2.getNumColumns(), false);
                    corr = new MatrixBlock(_tmpVal2.getNumRows(), _tmpVal2.getNumColumns(), false);
                }
                if (_aggOp.correctionExists)
                    OperationsOnMatrixValues.incrementalAggregation(outVal, corr, _tmpVal2, _aggOp, true);
                else
                    OperationsOnMatrixValues.incrementalAggregation(outVal, null, _tmpVal2, _aggOp, true);
            }
        }
    }
}