Examples with MatrixValue org.apache.sysml.runtime.matrix.data.MatrixValue used on opensource projects

Example 1 with MatrixValue

use of org.apache.sysml.runtime.matrix.data.MatrixValue in project incubator-systemml by apache.

the class AggregateBinaryInstruction method processMapMultInstruction.

/**
 * Helper function to perform map-side matrix-matrix multiplication.
 *
 * @param valueClass matrix value class
 * @param cachedValues cached value map
 * @param in1 indexed matrix value 1
 * @param in2 indexed matrix value 2
 * @param blockRowFactor ?
 * @param blockColFactor ?
 */
private void processMapMultInstruction(Class<? extends MatrixValue> valueClass, CachedValueMap cachedValues, IndexedMatrixValue in1, IndexedMatrixValue in2, int blockRowFactor, int blockColFactor) {
    boolean removeOutput = true;
    if (_cacheType.isRight()) {
        DistributedCacheInput dcInput = MRBaseForCommonInstructions.dcValues.get(input2);
        long in2_cols = dcInput.getNumCols();
        long in2_colBlocks = (long) Math.ceil(((double) in2_cols) / dcInput.getNumColsPerBlock());
        for (int bidx = 1; bidx <= in2_colBlocks; bidx++) {
            // Matrix multiply A[i,k] %*% B[k,bid]
            // Setup input2 block
            IndexedMatrixValue in2Block = dcInput.getDataBlock((int) in1.getIndexes().getColumnIndex(), bidx);
            MatrixValue in2BlockValue = in2Block.getValue();
            MatrixIndexes in2BlockIndex = in2Block.getIndexes();
            // allocate space for the output value
            IndexedMatrixValue out = cachedValues.holdPlace(output, valueClass);
            // process instruction
            OperationsOnMatrixValues.performAggregateBinary(in1.getIndexes(), (MatrixBlock) in1.getValue(), in2BlockIndex, (MatrixBlock) in2BlockValue, out.getIndexes(), (MatrixBlock) out.getValue(), ((AggregateBinaryOperator) optr));
            removeOutput &= (!_outputEmptyBlocks && out.getValue().isEmpty());
        }
    } else {
        DistributedCacheInput dcInput = MRBaseForCommonInstructions.dcValues.get(input1);
        long in1_rows = dcInput.getNumRows();
        long in1_rowsBlocks = (long) Math.ceil(((double) in1_rows) / dcInput.getNumRowsPerBlock());
        for (int bidx = 1; bidx <= in1_rowsBlocks; bidx++) {
            // Matrix multiply A[i,k] %*% B[k,bid]
            // Setup input2 block
            IndexedMatrixValue in1Block = dcInput.getDataBlock(bidx, (int) in2.getIndexes().getRowIndex());
            MatrixValue in1BlockValue = in1Block.getValue();
            MatrixIndexes in1BlockIndex = in1Block.getIndexes();
            // allocate space for the output value
            IndexedMatrixValue out = cachedValues.holdPlace(output, valueClass);
            // process instruction
            OperationsOnMatrixValues.performAggregateBinary(in1BlockIndex, (MatrixBlock) in1BlockValue, in2.getIndexes(), (MatrixBlock) in2.getValue(), out.getIndexes(), (MatrixBlock) out.getValue(), ((AggregateBinaryOperator) optr));
            removeOutput &= (!_outputEmptyBlocks && out.getValue().isEmpty());
        }
    }
    // empty block output filter (enabled by compiler consumer operation is in CP)
    if (removeOutput)
        cachedValues.remove(output);
}

Example 2 with MatrixValue

use of org.apache.sysml.runtime.matrix.data.MatrixValue in project incubator-systemml by apache.

the class QuaternaryInstruction method processInstruction.

@Override
public void processInstruction(Class<? extends MatrixValue> valueClass, CachedValueMap cachedValues, IndexedMatrixValue tempValue, IndexedMatrixValue zeroInput, int blockRowFactor, int blockColFactor) {
    QuaternaryOperator qop = (QuaternaryOperator) optr;
    ArrayList<IndexedMatrixValue> blkList = cachedValues.get(_input1);
    if (blkList != null)
        for (IndexedMatrixValue imv : blkList) {
            // Step 1: prepare inputs and output
            if (imv == null)
                continue;
            MatrixIndexes inIx = imv.getIndexes();
            MatrixBlock inVal = (MatrixBlock) imv.getValue();
            // allocate space for the output value
            IndexedMatrixValue iout = null;
            if (output == _input1)
                iout = tempValue;
            else
                iout = cachedValues.holdPlace(output, valueClass);
            MatrixIndexes outIx = iout.getIndexes();
            MatrixValue outVal = iout.getValue();
            // Step 2: get remaining inputs: Wij, Ui, Vj
            MatrixBlock Xij = inVal;
            // get Wij if existing (null of WeightsType.NONE or WSigmoid any type)
            IndexedMatrixValue iWij = (_input4 != -1) ? cachedValues.getFirst(_input4) : null;
            MatrixValue Wij = (iWij != null) ? iWij.getValue() : null;
            if (null == Wij && qop.hasFourInputs()) {
                MatrixBlock mb = new MatrixBlock(1, 1, false);
                String[] parts = InstructionUtils.getInstructionParts(instString);
                mb.quickSetValue(0, 0, Double.valueOf(parts[4]));
                Wij = mb;
            }
            // get Ui and Vj, potentially through distributed cache
            MatrixValue Ui = // U
            (!_cacheU) ? // U
            cachedValues.getFirst(_input2).getValue() : MRBaseForCommonInstructions.dcValues.get(_input2).getDataBlock((int) inIx.getRowIndex(), 1).getValue();
            MatrixValue Vj = // t(V)
            (!_cacheV) ? // t(V)
            cachedValues.getFirst(_input3).getValue() : MRBaseForCommonInstructions.dcValues.get(_input3).getDataBlock((int) inIx.getColumnIndex(), 1).getValue();
            // handle special input case: //V through shuffle -> t(V)
            if (Ui.getNumColumns() != Vj.getNumColumns()) {
                Vj = LibMatrixReorg.reorg((MatrixBlock) Vj, new MatrixBlock(Vj.getNumColumns(), Vj.getNumRows(), Vj.isInSparseFormat()), new ReorgOperator(SwapIndex.getSwapIndexFnObject()));
            }
            // Step 3: process instruction
            Xij.quaternaryOperations(qop, (MatrixBlock) Ui, (MatrixBlock) Vj, (MatrixBlock) Wij, (MatrixBlock) outVal);
            if (qop.wtype1 != null || qop.wtype4 != null)
                // wsloss
                outIx.setIndexes(1, 1);
            else if (qop.wtype2 != null || qop.wtype5 != null || qop.wtype3 != null && qop.wtype3.isBasic())
                // wsigmoid/wdivmm-basic
                outIx.setIndexes(inIx);
            else {
                // wdivmm
                boolean left = qop.wtype3.isLeft();
                outIx.setIndexes(left ? inIx.getColumnIndex() : inIx.getRowIndex(), 1);
            }
            // put the output value in the cache
            if (iout == tempValue)
                cachedValues.add(output, iout);
        }
}

Example 3 with MatrixValue

use of org.apache.sysml.runtime.matrix.data.MatrixValue in project incubator-systemml by apache.

the class ReplicateInstruction method processInstruction.

@Override
public void processInstruction(Class<? extends MatrixValue> valueClass, CachedValueMap cachedValues, IndexedMatrixValue tempValue, IndexedMatrixValue zeroInput, int blockRowFactor, int blockColFactor) {
    ArrayList<IndexedMatrixValue> blkList = cachedValues.get(input);
    if (blkList != null) {
        for (IndexedMatrixValue in : blkList) {
            if (in == null)
                continue;
            // allocate space for the output value
            IndexedMatrixValue out;
            if (input == output)
                out = tempValue;
            else
                out = cachedValues.holdPlace(output, valueClass);
            // process instruction
            MatrixIndexes inIx = in.getIndexes();
            MatrixValue inVal = in.getValue();
            if (// replicate columns
            _repCols) {
                // (e.g., M is Nx2700, blocksize=1000 -> numRep 2 because original block passed to index 1)
                if (// blocksize should be 1000 or similar
                blockColFactor <= 1)
                    LOG.warn("Block size of input matrix is: brlen=" + blockRowFactor + ", bclen=" + blockColFactor + ".");
                long numRep = (long) Math.ceil((double) _lenM / blockColFactor) - 1;
                // hence the memory overhead is very small)
                for (long i = 0; i < numRep; i++) {
                    IndexedMatrixValue repV = cachedValues.holdPlace(output, valueClass);
                    MatrixIndexes repIX = repV.getIndexes();
                    repIX.setIndexes(inIx.getRowIndex(), 2 + i);
                    repV.set(repIX, inVal);
                }
                // output original block
                out.set(inIx, inVal);
            } else // replicate rows
            {
                // (e.g., M is Nx2700, blocksize=1000 -> numRep 2 because original block passed to index 1)
                if (// blocksize should be 1000 or similar
                blockRowFactor <= 1)
                    LOG.warn("Block size of input matrix is: brlen=" + blockRowFactor + ", bclen=" + blockColFactor + ".");
                long numRep = (long) Math.ceil((double) _lenM / blockRowFactor) - 1;
                // hence the memory overhead is very small)
                for (long i = 0; i < numRep; i++) {
                    IndexedMatrixValue repV = cachedValues.holdPlace(output, valueClass);
                    MatrixIndexes repIX = repV.getIndexes();
                    repIX.setIndexes(2 + i, inIx.getColumnIndex());
                    repV.set(repIX, inVal);
                }
                // output original block
                out.set(inIx, inVal);
            }
            // put the output value in the cache
            if (out == tempValue)
                cachedValues.add(output, out);
        }
    }
}

Example 4 with MatrixValue

use of org.apache.sysml.runtime.matrix.data.MatrixValue in project incubator-systemml by apache.

the class MapperBase method commonMap.

@SuppressWarnings("unchecked")
protected void commonMap(Writable rawKey, Writable rawValue, OutputCollector<Writable, Writable> out, Reporter reporter) throws IOException {
    long start = System.currentTimeMillis();
    // for each representative matrix, read the record and apply instructions
    for (int i = 0; i < representativeMatrixes.size(); i++) {
        byte thisMatrix = representativeMatrixes.get(i);
        // convert the record into the right format for the representative matrix
        inputConverter.setBlockSize(brlens[i], bclens[i]);
        inputConverter.convert(rawKey, rawValue);
        // apply unary instructions on the converted indexes and values
        while (inputConverter.hasNext()) {
            Pair<MatrixIndexes, MatrixValue> pair = inputConverter.next();
            MatrixIndexes indexes = pair.getKey();
            MatrixValue value = pair.getValue();
            checkValidity(indexes, value, i);
            // put the input in the cache
            cachedValues.reset();
            cachedValues.set(thisMatrix, indexes, value);
            // special operations for individual mapp type
            specialOperationsForActualMap(i, out, reporter);
        }
    }
    reporter.incrCounter(Counters.MAP_TIME, System.currentTimeMillis() - start);
}

Example 5 with MatrixValue

use of org.apache.sysml.runtime.matrix.data.MatrixValue in project incubator-systemml by apache.

the class ReblockMapper method processReblockInMapperAndOutput.

protected void processReblockInMapperAndOutput(int index, OutputCollector<Writable, Writable> out) throws IOException {
    for (ReblockInstruction ins : reblock_instructions.get(index)) {
        ArrayList<IndexedMatrixValue> ixvList = cachedValues.get(ins.input);
        if (ixvList != null) {
            for (IndexedMatrixValue inValue : ixvList) {
                if (inValue == null)
                    continue;
                // get buffer
                ReblockBuffer rbuff = buffer.get(ins.output);
                if (rbuff == null) {
                    MatrixCharacteristics mc = dimensionsOut.get(ins.output);
                    rbuff = new ReblockBuffer(buffersize, mc.getRows(), mc.getCols(), ins.brlen, ins.bclen);
                    buffer.put(ins.output, rbuff);
                }
                // append cells and flush buffer if required
                MatrixValue mval = inValue.getValue();
                if (mval instanceof MatrixBlock) {
                    MatrixIndexes inIx = inValue.getIndexes();
                    MatrixCharacteristics mc = dimensionsIn.get(ins.input);
                    long row_offset = (inIx.getRowIndex() - 1) * mc.getRowsPerBlock() + 1;
                    long col_offset = (inIx.getColumnIndex() - 1) * mc.getColsPerBlock() + 1;
                    // append entire block incl. flush on demand
                    rbuff.appendBlock(row_offset, col_offset, (MatrixBlock) mval, ins.output, out);
                } else // if( mval instanceof MatrixCell )
                {
                    rbuff.appendCell(inValue.getIndexes().getRowIndex(), inValue.getIndexes().getColumnIndex(), ((MatrixCell) mval).getValue());
                    // flush buffer if necessary
                    if (rbuff.getSize() >= rbuff.getCapacity())
                        rbuff.flushBuffer(ins.output, out);
                }
            }
        }
    }
}