Examples with IndexedMatrixValue org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue used on opensource projects

Example 21 with IndexedMatrixValue

use of org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue in project systemml by apache.

the class OperationsOnMatrixValues method performShift.

public static void performShift(IndexedMatrixValue in, IndexRange ixrange, int brlen, int bclen, long rlen, long clen, ArrayList<IndexedMatrixValue> outlist) {
    MatrixIndexes ix = in.getIndexes();
    MatrixBlock mb = (MatrixBlock) in.getValue();
    long start_lhs_globalRowIndex = ixrange.rowStart + (ix.getRowIndex() - 1) * brlen;
    long start_lhs_globalColIndex = ixrange.colStart + (ix.getColumnIndex() - 1) * bclen;
    long end_lhs_globalRowIndex = start_lhs_globalRowIndex + mb.getNumRows() - 1;
    long end_lhs_globalColIndex = start_lhs_globalColIndex + mb.getNumColumns() - 1;
    long start_lhs_rowIndex = UtilFunctions.computeBlockIndex(start_lhs_globalRowIndex, brlen);
    long end_lhs_rowIndex = UtilFunctions.computeBlockIndex(end_lhs_globalRowIndex, brlen);
    long start_lhs_colIndex = UtilFunctions.computeBlockIndex(start_lhs_globalColIndex, bclen);
    long end_lhs_colIndex = UtilFunctions.computeBlockIndex(end_lhs_globalColIndex, bclen);
    for (long leftRowIndex = start_lhs_rowIndex; leftRowIndex <= end_lhs_rowIndex; leftRowIndex++) {
        for (long leftColIndex = start_lhs_colIndex; leftColIndex <= end_lhs_colIndex; leftColIndex++) {
            // Calculate global index of right hand side block
            long lhs_rl = Math.max((leftRowIndex - 1) * brlen + 1, start_lhs_globalRowIndex);
            long lhs_ru = Math.min(leftRowIndex * brlen, end_lhs_globalRowIndex);
            long lhs_cl = Math.max((leftColIndex - 1) * bclen + 1, start_lhs_globalColIndex);
            long lhs_cu = Math.min(leftColIndex * bclen, end_lhs_globalColIndex);
            int lhs_lrl = UtilFunctions.computeCellInBlock(lhs_rl, brlen);
            int lhs_lru = UtilFunctions.computeCellInBlock(lhs_ru, brlen);
            int lhs_lcl = UtilFunctions.computeCellInBlock(lhs_cl, bclen);
            int lhs_lcu = UtilFunctions.computeCellInBlock(lhs_cu, bclen);
            long rhs_rl = lhs_rl - ixrange.rowStart + 1;
            long rhs_ru = rhs_rl + (lhs_ru - lhs_rl);
            long rhs_cl = lhs_cl - ixrange.colStart + 1;
            long rhs_cu = rhs_cl + (lhs_cu - lhs_cl);
            int rhs_lrl = UtilFunctions.computeCellInBlock(rhs_rl, brlen);
            int rhs_lru = UtilFunctions.computeCellInBlock(rhs_ru, brlen);
            int rhs_lcl = UtilFunctions.computeCellInBlock(rhs_cl, bclen);
            int rhs_lcu = UtilFunctions.computeCellInBlock(rhs_cu, bclen);
            MatrixBlock slicedRHSBlk = mb.slice(rhs_lrl, rhs_lru, rhs_lcl, rhs_lcu, new MatrixBlock());
            int lbrlen = UtilFunctions.computeBlockSize(rlen, leftRowIndex, brlen);
            int lbclen = UtilFunctions.computeBlockSize(clen, leftColIndex, bclen);
            MatrixBlock resultBlock = new MatrixBlock(lbrlen, lbclen, false);
            resultBlock = resultBlock.leftIndexingOperations(slicedRHSBlk, lhs_lrl, lhs_lru, lhs_lcl, lhs_lcu, null, UpdateType.COPY);
            outlist.add(new IndexedMatrixValue(new MatrixIndexes(leftRowIndex, leftColIndex), resultBlock));
        }
    }
}

Example 22 with IndexedMatrixValue

use of org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue in project systemml by apache.

the class ReaderBinaryBlock method readBinaryBlockMatrixBlocksFromHDFS.

private static void readBinaryBlockMatrixBlocksFromHDFS(Path path, JobConf job, FileSystem fs, Collection<IndexedMatrixValue> dest, long rlen, long clen, int brlen, int bclen) throws IOException {
    MatrixIndexes key = new MatrixIndexes();
    MatrixBlock value = new MatrixBlock();
    // set up preferred custom serialization framework for binary block format
    if (MRJobConfiguration.USE_BINARYBLOCK_SERIALIZATION)
        MRJobConfiguration.addBinaryBlockSerializationFramework(job);
    for (// 1..N files
    Path lpath : // 1..N files
    IOUtilFunctions.getSequenceFilePaths(fs, path)) {
        // directly read from sequence files (individual partfiles)
        SequenceFile.Reader reader = new SequenceFile.Reader(job, SequenceFile.Reader.file(lpath));
        try {
            while (reader.next(key, value)) {
                int row_offset = (int) (key.getRowIndex() - 1) * brlen;
                int col_offset = (int) (key.getColumnIndex() - 1) * bclen;
                int rows = value.getNumRows();
                int cols = value.getNumColumns();
                // bound check per block
                if (row_offset + rows < 0 || row_offset + rows > rlen || col_offset + cols < 0 || col_offset + cols > clen) {
                    throw new IOException("Matrix block [" + (row_offset + 1) + ":" + (row_offset + rows) + "," + (col_offset + 1) + ":" + (col_offset + cols) + "] " + "out of overall matrix range [1:" + rlen + ",1:" + clen + "].");
                }
                // copy block to result
                dest.add(new IndexedMatrixValue(new MatrixIndexes(key), new MatrixBlock(value)));
            }
        } finally {
            IOUtilFunctions.closeSilently(reader);
        }
    }
}

Example 23 with IndexedMatrixValue

use of org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue in project incubator-systemml by apache.

the class TernaryInstruction method processInstruction.

public void processInstruction(Class<? extends MatrixValue> valueClass, CachedValueMap cachedValues, IndexedMatrixValue zeroInput, HashMap<Byte, CTableMap> resultMaps, HashMap<Byte, MatrixBlock> resultBlocks, int blockRowFactor, int blockColFactor) throws DMLRuntimeException {
    IndexedMatrixValue in1, in2, in3 = null;
    in1 = cachedValues.getFirst(input1);
    CTableMap ctableResult = null;
    MatrixBlock ctableResultBlock = null;
    if (knownOutputDims()) {
        if (resultBlocks != null) {
            ctableResultBlock = resultBlocks.get(output);
            if (ctableResultBlock == null) {
                // From MR, output of ctable is set to be sparse since it is built from a single input block.
                ctableResultBlock = new MatrixBlock((int) _outputDim1, (int) _outputDim2, true);
                resultBlocks.put(output, ctableResultBlock);
            }
        } else {
            throw new DMLRuntimeException("Unexpected error in processing table instruction.");
        }
    } else {
        //prepare aggregation maps
        ctableResult = resultMaps.get(output);
        if (ctableResult == null) {
            ctableResult = new CTableMap();
            resultMaps.put(output, ctableResult);
        }
    }
    //get inputs and process instruction
    switch(_op) {
        case CTABLE_TRANSFORM:
            {
                in2 = cachedValues.getFirst(input2);
                in3 = cachedValues.getFirst(input3);
                if (in1 == null || in2 == null || in3 == null)
                    return;
                OperationsOnMatrixValues.performTernary(in1.getIndexes(), in1.getValue(), in2.getIndexes(), in2.getValue(), in3.getIndexes(), in3.getValue(), ctableResult, ctableResultBlock, optr);
                break;
            }
        case CTABLE_TRANSFORM_SCALAR_WEIGHT:
            {
                // 3rd input is a scalar
                in2 = cachedValues.getFirst(input2);
                if (in1 == null || in2 == null)
                    return;
                OperationsOnMatrixValues.performTernary(in1.getIndexes(), in1.getValue(), in2.getIndexes(), in2.getValue(), scalar_input3, ctableResult, ctableResultBlock, optr);
                break;
            }
        case CTABLE_EXPAND_SCALAR_WEIGHT:
            {
                // 2nd and 3rd input is a scalar
                if (in1 == null)
                    return;
                OperationsOnMatrixValues.performTernary(in1.getIndexes(), in1.getValue(), scalar_input2, (scalar_input3 == 1), blockRowFactor, ctableResult, ctableResultBlock, optr);
                break;
            }
        case CTABLE_TRANSFORM_HISTOGRAM:
            {
                // 2nd and 3rd inputs are scalars
                if (in1 == null)
                    return;
                OperationsOnMatrixValues.performTernary(in1.getIndexes(), in1.getValue(), scalar_input2, scalar_input3, ctableResult, ctableResultBlock, optr);
                break;
            }
        case CTABLE_TRANSFORM_WEIGHTED_HISTOGRAM:
            {
                // 2nd and 3rd inputs are scalars
                in3 = cachedValues.getFirst(input3);
                if (in1 == null || in3 == null)
                    return;
                OperationsOnMatrixValues.performTernary(in1.getIndexes(), in1.getValue(), scalar_input2, in3.getIndexes(), in3.getValue(), ctableResult, ctableResultBlock, optr);
                break;
            }
        default:
            throw new DMLRuntimeException("Unrecognized opcode in Tertiary Instruction: " + instString);
    }
}

Example 24 with IndexedMatrixValue

use of org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue in project incubator-systemml by apache.

the class AppendMInstruction method processInstruction.

@Override
public void processInstruction(Class<? extends MatrixValue> valueClass, CachedValueMap cachedValues, IndexedMatrixValue tempValue, IndexedMatrixValue zeroInput, int blockRowFactor, int blockColFactor) {
    ArrayList<IndexedMatrixValue> blkList = cachedValues.get(input1);
    if (blkList == null)
        return;
    // right now this only deals with appending matrix with number of column <= blockColFactor
    for (IndexedMatrixValue in1 : blkList) {
        if (in1 == null)
            continue;
        // check for boundary block w/ awareness of zero rows or columns
        int blen = _cbind ? blockColFactor : blockRowFactor;
        long lastBlockColIndex = Math.max((long) Math.ceil((double) _offset / blen), 1);
        // case 1: pass through of non-boundary blocks
        MatrixIndexes ix = in1.getIndexes();
        if ((_cbind ? ix.getColumnIndex() : ix.getRowIndex()) != lastBlockColIndex) {
            cachedValues.add(output, in1);
        } else // case 2: pass through full input block and rhs block
        if (_cbind && in1.getValue().getNumColumns() == blen || !_cbind && in1.getValue().getNumRows() == blen) {
            // output lhs block
            cachedValues.add(output, in1);
            // output shallow copy of rhs block
            DistributedCacheInput dcInput = MRBaseForCommonInstructions.dcValues.get(input2);
            if (_cbind) {
                cachedValues.add(output, new IndexedMatrixValue(new MatrixIndexes(ix.getRowIndex(), ix.getColumnIndex() + 1), dcInput.getDataBlock((int) ix.getRowIndex(), 1).getValue()));
            } else {
                cachedValues.add(output, new IndexedMatrixValue(new MatrixIndexes(ix.getRowIndex() + 1, ix.getColumnIndex()), dcInput.getDataBlock(1, (int) ix.getColumnIndex()).getValue()));
            }
        } else // case 3: append operation on boundary block
        {
            DistributedCacheInput dcInput = MRBaseForCommonInstructions.dcValues.get(input2);
            // allocate space for the output value
            ArrayList<IndexedMatrixValue> outlist = new ArrayList<>(2);
            IndexedMatrixValue first = cachedValues.holdPlace(output, valueClass);
            first.getIndexes().setIndexes(ix);
            outlist.add(first);
            MatrixValue value_in2 = null;
            if (_cbind) {
                value_in2 = dcInput.getDataBlock((int) ix.getRowIndex(), 1).getValue();
                if (in1.getValue().getNumColumns() + value_in2.getNumColumns() > blen) {
                    IndexedMatrixValue second = cachedValues.holdPlace(output, valueClass);
                    second.getIndexes().setIndexes(ix.getRowIndex(), ix.getColumnIndex() + 1);
                    outlist.add(second);
                }
            } else {
                // rbind
                value_in2 = dcInput.getDataBlock(1, (int) ix.getColumnIndex()).getValue();
                if (in1.getValue().getNumRows() + value_in2.getNumRows() > blen) {
                    IndexedMatrixValue second = cachedValues.holdPlace(output, valueClass);
                    second.getIndexes().setIndexes(ix.getRowIndex() + 1, ix.getColumnIndex());
                    outlist.add(second);
                }
            }
            OperationsOnMatrixValues.performAppend(in1.getValue(), value_in2, outlist, blockRowFactor, blockColFactor, _cbind, true, 0);
        }
    }
}

Example 25 with IndexedMatrixValue

use of org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue in project incubator-systemml by apache.

the class AppendRInstruction method processInstruction.

@Override
public void processInstruction(Class<? extends MatrixValue> valueClass, CachedValueMap cachedValues, IndexedMatrixValue tempValue, IndexedMatrixValue zeroInput, int brlen, int bclen) {
    // get both inputs
    IndexedMatrixValue left = cachedValues.getFirst(input1);
    IndexedMatrixValue right = cachedValues.getFirst(input2);
    // check non-existing block
    if (left == null || right == null)
        throw new DMLRuntimeException("Missing append input: isNull(left): " + (left == null) + ", isNull(right): " + (right == null));
    // core append operation
    MatrixBlock mbLeft = (MatrixBlock) left.getValue();
    MatrixBlock mbRight = (MatrixBlock) right.getValue();
    MatrixBlock ret = mbLeft.append(mbRight, new MatrixBlock(), _cbind);
    // put result into cache
    cachedValues.add(output, new IndexedMatrixValue(left.getIndexes(), ret));
}