Examples with IndexedMatrixValue org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue used on opensource projects

Example 46 with IndexedMatrixValue

use of org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue in project systemml by apache.

the class PMMJMRInstruction method processInstruction.

@Override
public void processInstruction(Class<? extends MatrixValue> valueClass, CachedValueMap cachedValues, IndexedMatrixValue tempValue, IndexedMatrixValue zeroInput, int blockRowFactor, int blockColFactor) {
    // get both matrix inputs (left side always permutation)
    DistributedCacheInput dcInput = MRBaseForCommonInstructions.dcValues.get(input1);
    IndexedMatrixValue in2 = cachedValues.getFirst(input2);
    IndexedMatrixValue in1 = dcInput.getDataBlock((int) in2.getIndexes().getRowIndex(), 1);
    MatrixBlock mb1 = (MatrixBlock) in1.getValue();
    MatrixBlock mb2 = (MatrixBlock) in2.getValue();
    // compute target block indexes
    long minPos = UtilFunctions.toLong(mb1.minNonZero());
    long maxPos = UtilFunctions.toLong(mb1.max());
    long rowIX1 = (minPos - 1) / blockRowFactor + 1;
    long rowIX2 = (maxPos - 1) / blockRowFactor + 1;
    boolean multipleOuts = (rowIX1 != rowIX2);
    if (// at least one row selected
    minPos >= 1) {
        // output sparsity estimate
        double spmb1 = OptimizerUtils.getSparsity(mb1.getNumRows(), 1, mb1.getNonZeros());
        long estnnz = (long) (spmb1 * mb2.getNonZeros());
        boolean sparse = MatrixBlock.evalSparseFormatInMemory(blockRowFactor, mb2.getNumColumns(), estnnz);
        // compute and allocate output blocks
        IndexedMatrixValue out1 = cachedValues.holdPlace(output, valueClass);
        IndexedMatrixValue out2 = multipleOuts ? cachedValues.holdPlace(output, valueClass) : null;
        out1.getValue().reset(blockRowFactor, mb2.getNumColumns(), sparse);
        if (out2 != null)
            out2.getValue().reset(UtilFunctions.computeBlockSize(_rlen, rowIX2, blockRowFactor), mb2.getNumColumns(), sparse);
        // compute core matrix permutation (assumes that out1 has default blocksize,
        // hence we do a meta data correction afterwards)
        mb1.permutationMatrixMultOperations(mb2, out1.getValue(), (out2 != null) ? out2.getValue() : null);
        ((MatrixBlock) out1.getValue()).setNumRows(UtilFunctions.computeBlockSize(_rlen, rowIX1, blockRowFactor));
        out1.getIndexes().setIndexes(rowIX1, in2.getIndexes().getColumnIndex());
        if (out2 != null)
            out2.getIndexes().setIndexes(rowIX2, in2.getIndexes().getColumnIndex());
        // empty block output filter (enabled by compiler consumer operation is in CP)
        if (!_outputEmptyBlocks && out1.getValue().isEmpty() && (out2 == null || out2.getValue().isEmpty())) {
            cachedValues.remove(output);
        }
    }
}

Example 47 with IndexedMatrixValue

use of org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue in project systemml by apache.

the class ParameterizedBuiltinMRInstruction method processInstruction.

@Override
public void processInstruction(Class<? extends MatrixValue> valueClass, CachedValueMap cachedValues, IndexedMatrixValue tempValue, IndexedMatrixValue zeroInput, int blockRowFactor, int blockColFactor) {
    ArrayList<IndexedMatrixValue> blkList = cachedValues.get(input);
    if (blkList != null)
        for (IndexedMatrixValue imv : blkList) {
            if (imv == null)
                continue;
            if (_opcode.equalsIgnoreCase("replace")) {
                MatrixValue in = imv.getValue();
                MatrixIndexes inIX = imv.getIndexes();
                // allocate space for the output value
                IndexedMatrixValue iout = null;
                if (output == input)
                    iout = tempValue;
                else
                    iout = cachedValues.holdPlace(output, valueClass);
                iout.getIndexes().setIndexes(inIX);
                MatrixValue out = iout.getValue();
                // process instruction
                in.replaceOperations(out, _pattern, _replace);
                // put the output value in the cache
                if (iout == tempValue)
                    cachedValues.add(output, iout);
            } else if (_opcode.equalsIgnoreCase("rexpand")) {
                // process instruction
                ArrayList<IndexedMatrixValue> out = new ArrayList<>();
                LibMatrixReorg.rexpand(imv, _max, _dirRows, _cast, _ignore, blockRowFactor, blockColFactor, out);
                // put the output values in the cache
                for (IndexedMatrixValue lout : out) cachedValues.add(output, lout);
            }
        }
}

Example 48 with IndexedMatrixValue

use of org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue in project systemml by apache.

the class RemoveEmptyMRInstruction method processInstruction.

@Override
public void processInstruction(Class<? extends MatrixValue> valueClass, CachedValueMap cachedValues, IndexedMatrixValue tempValue, IndexedMatrixValue zeroInput, int blockRowFactor, int blockColFactor) {
    // get input and offsets
    IndexedMatrixValue inData = cachedValues.getFirst(input1);
    IndexedMatrixValue inOffset = cachedValues.getFirst(input2);
    MatrixIndexes ix = inData.getIndexes();
    MatrixValue mb = inData.getValue();
    if (_len > 0) {
        // execute remove empty operations
        ArrayList<IndexedMatrixValue> out = new ArrayList<>();
        LibMatrixReorg.rmempty(inData, inOffset, _rmRows, _len, blockRowFactor, blockColFactor, out);
        // put results into cache map
        for (IndexedMatrixValue imv : out) cachedValues.add(output, imv);
    } else {
        int n = _emptyRet ? 1 : 0;
        cachedValues.add(output, new IndexedMatrixValue(new MatrixIndexes(_rmRows ? 1 : ix.getRowIndex(), _rmRows ? ix.getColumnIndex() : 1), new MatrixBlock(_rmRows ? n : mb.getNumRows(), _rmRows ? mb.getNumColumns() : n, false)));
    }
}

Example 49 with IndexedMatrixValue

use of org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue in project systemml by apache.

the class ReorgInstruction method processInstruction.

@Override
public void processInstruction(Class<? extends MatrixValue> valueClass, CachedValueMap cachedValues, IndexedMatrixValue tempValue, IndexedMatrixValue zeroInput, int blockRowFactor, int blockColFactor) {
    ArrayList<IndexedMatrixValue> blkList = cachedValues.get(input);
    if (blkList != null)
        for (IndexedMatrixValue in : blkList) {
            if (in == null)
                continue;
            int startRow = 0, startColumn = 0, length = 0;
            // process instruction
            if (((ReorgOperator) optr).fn instanceof DiagIndex) {
                // special diag handling (overloaded, size-dependent operation; hence decided during runtime)
                boolean V2M = (_mcIn.getRows() == 1 || _mcIn.getCols() == 1);
                // input can be row/column vector
                long rlen = Math.max(_mcIn.getRows(), _mcIn.getCols());
                // Note: for M2V we directly skip non-diagonal blocks block
                if (V2M || in.getIndexes().getRowIndex() == in.getIndexes().getColumnIndex()) {
                    if (V2M) {
                        // allocate space for the output value
                        IndexedMatrixValue out = cachedValues.holdPlace(output, valueClass);
                        OperationsOnMatrixValues.performReorg(in.getIndexes(), in.getValue(), out.getIndexes(), out.getValue(), ((ReorgOperator) optr), startRow, startColumn, length);
                        // (only for block representation)
                        if (_outputEmptyBlocks && valueClass.equals(MatrixBlock.class)) {
                            // row index is equal to the col index
                            long diagIndex = out.getIndexes().getRowIndex();
                            long brlen = Math.max(_mcIn.getRowsPerBlock(), _mcIn.getColsPerBlock());
                            long numRowBlocks = (rlen / brlen) + ((rlen % brlen != 0) ? 1 : 0);
                            for (long rc = 1; rc <= numRowBlocks; rc++) {
                                // prevent duplicate output
                                if (rc == diagIndex)
                                    continue;
                                IndexedMatrixValue emptyIndexValue = cachedValues.holdPlace(output, valueClass);
                                int lbrlen = (int) ((rc * brlen <= rlen) ? brlen : rlen % brlen);
                                emptyIndexValue.getIndexes().setIndexes(rc, diagIndex);
                                emptyIndexValue.getValue().reset(lbrlen, out.getValue().getNumColumns(), true);
                            }
                        }
                    } else // M2V
                    {
                        // allocate space for the output value
                        IndexedMatrixValue out = cachedValues.holdPlace(output, valueClass);
                        // compute matrix indexes
                        out.getIndexes().setIndexes(in.getIndexes().getRowIndex(), 1);
                        // compute result block
                        in.getValue().reorgOperations((ReorgOperator) optr, out.getValue(), startRow, startColumn, length);
                    }
                }
            } else if (((ReorgOperator) optr).fn instanceof RevIndex) {
                // execute reverse operation
                ArrayList<IndexedMatrixValue> out = new ArrayList<>();
                LibMatrixReorg.rev(in, _mcIn.getRows(), _mcIn.getRowsPerBlock(), out);
                // output indexed matrix values
                for (IndexedMatrixValue outblk : out) cachedValues.add(output, outblk);
            } else // general case (e.g., transpose)
            {
                // allocate space for the output value
                IndexedMatrixValue out = cachedValues.holdPlace(output, valueClass);
                OperationsOnMatrixValues.performReorg(in.getIndexes(), in.getValue(), out.getIndexes(), out.getValue(), ((ReorgOperator) optr), startRow, startColumn, length);
            }
        }
}

Example 50 with IndexedMatrixValue

use of org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue in project systemml by apache.

the class ReplicateInstruction method processInstruction.

@Override
public void processInstruction(Class<? extends MatrixValue> valueClass, CachedValueMap cachedValues, IndexedMatrixValue tempValue, IndexedMatrixValue zeroInput, int blockRowFactor, int blockColFactor) {
    ArrayList<IndexedMatrixValue> blkList = cachedValues.get(input);
    if (blkList != null) {
        for (IndexedMatrixValue in : blkList) {
            if (in == null)
                continue;
            // allocate space for the output value
            IndexedMatrixValue out;
            if (input == output)
                out = tempValue;
            else
                out = cachedValues.holdPlace(output, valueClass);
            // process instruction
            MatrixIndexes inIx = in.getIndexes();
            MatrixValue inVal = in.getValue();
            if (// replicate columns
            _repCols) {
                // (e.g., M is Nx2700, blocksize=1000 -> numRep 2 because original block passed to index 1)
                if (// blocksize should be 1000 or similar
                blockColFactor <= 1)
                    LOG.warn("Block size of input matrix is: brlen=" + blockRowFactor + ", bclen=" + blockColFactor + ".");
                long numRep = (long) Math.ceil((double) _lenM / blockColFactor) - 1;
                // hence the memory overhead is very small)
                for (long i = 0; i < numRep; i++) {
                    IndexedMatrixValue repV = cachedValues.holdPlace(output, valueClass);
                    MatrixIndexes repIX = repV.getIndexes();
                    repIX.setIndexes(inIx.getRowIndex(), 2 + i);
                    repV.set(repIX, inVal);
                }
                // output original block
                out.set(inIx, inVal);
            } else // replicate rows
            {
                // (e.g., M is Nx2700, blocksize=1000 -> numRep 2 because original block passed to index 1)
                if (// blocksize should be 1000 or similar
                blockRowFactor <= 1)
                    LOG.warn("Block size of input matrix is: brlen=" + blockRowFactor + ", bclen=" + blockColFactor + ".");
                long numRep = (long) Math.ceil((double) _lenM / blockRowFactor) - 1;
                // hence the memory overhead is very small)
                for (long i = 0; i < numRep; i++) {
                    IndexedMatrixValue repV = cachedValues.holdPlace(output, valueClass);
                    MatrixIndexes repIX = repV.getIndexes();
                    repIX.setIndexes(2 + i, inIx.getColumnIndex());
                    repV.set(repIX, inVal);
                }
                // output original block
                out.set(inIx, inVal);
            }
            // put the output value in the cache
            if (out == tempValue)
                cachedValues.add(output, out);
        }
    }
}