Examples with IndexedMatrixValue org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue used on opensource projects

Example 6 with IndexedMatrixValue

use of org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue in project incubator-systemml by apache.

the class ReplicateInstruction method processInstruction.

@Override
public void processInstruction(Class<? extends MatrixValue> valueClass, CachedValueMap cachedValues, IndexedMatrixValue tempValue, IndexedMatrixValue zeroInput, int blockRowFactor, int blockColFactor) {
    ArrayList<IndexedMatrixValue> blkList = cachedValues.get(input);
    if (blkList != null) {
        for (IndexedMatrixValue in : blkList) {
            if (in == null)
                continue;
            // allocate space for the output value
            IndexedMatrixValue out;
            if (input == output)
                out = tempValue;
            else
                out = cachedValues.holdPlace(output, valueClass);
            // process instruction
            MatrixIndexes inIx = in.getIndexes();
            MatrixValue inVal = in.getValue();
            if (// replicate columns
            _repCols) {
                // (e.g., M is Nx2700, blocksize=1000 -> numRep 2 because original block passed to index 1)
                if (// blocksize should be 1000 or similar
                blockColFactor <= 1)
                    LOG.warn("Block size of input matrix is: brlen=" + blockRowFactor + ", bclen=" + blockColFactor + ".");
                long numRep = (long) Math.ceil((double) _lenM / blockColFactor) - 1;
                // hence the memory overhead is very small)
                for (long i = 0; i < numRep; i++) {
                    IndexedMatrixValue repV = cachedValues.holdPlace(output, valueClass);
                    MatrixIndexes repIX = repV.getIndexes();
                    repIX.setIndexes(inIx.getRowIndex(), 2 + i);
                    repV.set(repIX, inVal);
                }
                // output original block
                out.set(inIx, inVal);
            } else // replicate rows
            {
                // (e.g., M is Nx2700, blocksize=1000 -> numRep 2 because original block passed to index 1)
                if (// blocksize should be 1000 or similar
                blockRowFactor <= 1)
                    LOG.warn("Block size of input matrix is: brlen=" + blockRowFactor + ", bclen=" + blockColFactor + ".");
                long numRep = (long) Math.ceil((double) _lenM / blockRowFactor) - 1;
                // hence the memory overhead is very small)
                for (long i = 0; i < numRep; i++) {
                    IndexedMatrixValue repV = cachedValues.holdPlace(output, valueClass);
                    MatrixIndexes repIX = repV.getIndexes();
                    repIX.setIndexes(2 + i, inIx.getColumnIndex());
                    repV.set(repIX, inVal);
                }
                // output original block
                out.set(inIx, inVal);
            }
            // put the output value in the cache
            if (out == tempValue)
                cachedValues.add(output, out);
        }
    }
}

Example 7 with IndexedMatrixValue

use of org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue in project incubator-systemml by apache.

the class TernaryInstruction method processInstruction.

@Override
public void processInstruction(Class<? extends MatrixValue> valueClass, CachedValueMap cachedValues, IndexedMatrixValue tempValue, IndexedMatrixValue zeroInput, int blockRowFactor, int blockColFactor) {
    MatrixBlock lm1 = input1.isMatrix() ? (MatrixBlock) cachedValues.getFirst(ixinput1).getValue() : m1;
    MatrixBlock lm2 = input2.isMatrix() ? (MatrixBlock) cachedValues.getFirst(ixinput2).getValue() : m2;
    MatrixBlock lm3 = input3.isMatrix() ? (MatrixBlock) cachedValues.getFirst(ixinput3).getValue() : m3;
    MatrixIndexes ixin = input1.isMatrix() ? cachedValues.getFirst(ixinput1).getIndexes() : input2.isMatrix() ? cachedValues.getFirst(ixinput2).getIndexes() : cachedValues.getFirst(ixinput3).getIndexes();
    // prepare output
    IndexedMatrixValue out = new IndexedMatrixValue(new MatrixIndexes(), new MatrixBlock());
    out.getIndexes().setIndexes(ixin);
    // process instruction
    TernaryOperator op = (TernaryOperator) optr;
    lm1.ternaryOperations(op, lm2, lm3, (MatrixBlock) out.getValue());
    // put the output value in the cache
    cachedValues.add(ixoutput, out);
}

Example 8 with IndexedMatrixValue

use of org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue in project incubator-systemml by apache.

the class ReaderBinaryBlock method readBinaryBlockMatrixBlocksFromHDFS.

private static void readBinaryBlockMatrixBlocksFromHDFS(Path path, JobConf job, FileSystem fs, Collection<IndexedMatrixValue> dest, long rlen, long clen, int brlen, int bclen) throws IOException {
    MatrixIndexes key = new MatrixIndexes();
    MatrixBlock value = new MatrixBlock();
    // set up preferred custom serialization framework for binary block format
    if (MRJobConfiguration.USE_BINARYBLOCK_SERIALIZATION)
        MRJobConfiguration.addBinaryBlockSerializationFramework(job);
    for (// 1..N files
    Path lpath : // 1..N files
    IOUtilFunctions.getSequenceFilePaths(fs, path)) {
        // directly read from sequence files (individual partfiles)
        SequenceFile.Reader reader = new SequenceFile.Reader(job, SequenceFile.Reader.file(lpath));
        try {
            while (reader.next(key, value)) {
                int row_offset = (int) (key.getRowIndex() - 1) * brlen;
                int col_offset = (int) (key.getColumnIndex() - 1) * bclen;
                int rows = value.getNumRows();
                int cols = value.getNumColumns();
                // bound check per block
                if (row_offset + rows < 0 || row_offset + rows > rlen || col_offset + cols < 0 || col_offset + cols > clen) {
                    throw new IOException("Matrix block [" + (row_offset + 1) + ":" + (row_offset + rows) + "," + (col_offset + 1) + ":" + (col_offset + cols) + "] " + "out of overall matrix range [1:" + rlen + ",1:" + clen + "].");
                }
                // copy block to result
                dest.add(new IndexedMatrixValue(new MatrixIndexes(key), new MatrixBlock(value)));
            }
        } finally {
            IOUtilFunctions.closeSilently(reader);
        }
    }
}

Example 9 with IndexedMatrixValue

use of org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue in project incubator-systemml by apache.

the class ReaderBinaryBlock method readIndexedMatrixBlocksFromHDFS.

public ArrayList<IndexedMatrixValue> readIndexedMatrixBlocksFromHDFS(String fname, long rlen, long clen, int brlen, int bclen) throws IOException, DMLRuntimeException {
    // allocate output matrix block collection
    ArrayList<IndexedMatrixValue> ret = new ArrayList<>();
    // prepare file access
    JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
    Path path = new Path((_localFS ? "file:///" : "") + fname);
    FileSystem fs = IOUtilFunctions.getFileSystem(path, job);
    // check existence and non-empty file
    checkValidInputFile(fs, path);
    // core read
    readBinaryBlockMatrixBlocksFromHDFS(path, job, fs, ret, rlen, clen, brlen, bclen);
    return ret;
}

Example 10 with IndexedMatrixValue

use of org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue in project incubator-systemml by apache.

the class OperationsOnMatrixValues method performSlice.

public static void performSlice(IndexedMatrixValue in, IndexRange ixrange, int brlen, int bclen, ArrayList<IndexedMatrixValue> outlist) {
    long cellIndexTopRow = UtilFunctions.computeCellIndex(in.getIndexes().getRowIndex(), brlen, 0);
    long cellIndexBottomRow = UtilFunctions.computeCellIndex(in.getIndexes().getRowIndex(), brlen, in.getValue().getNumRows() - 1);
    long cellIndexLeftCol = UtilFunctions.computeCellIndex(in.getIndexes().getColumnIndex(), bclen, 0);
    long cellIndexRightCol = UtilFunctions.computeCellIndex(in.getIndexes().getColumnIndex(), bclen, in.getValue().getNumColumns() - 1);
    long cellIndexOverlapTop = Math.max(cellIndexTopRow, ixrange.rowStart);
    long cellIndexOverlapBottom = Math.min(cellIndexBottomRow, ixrange.rowEnd);
    long cellIndexOverlapLeft = Math.max(cellIndexLeftCol, ixrange.colStart);
    long cellIndexOverlapRight = Math.min(cellIndexRightCol, ixrange.colEnd);
    // check if block is outside the indexing range
    if (cellIndexOverlapTop > cellIndexOverlapBottom || cellIndexOverlapLeft > cellIndexOverlapRight) {
        return;
    }
    IndexRange tmpRange = new IndexRange(UtilFunctions.computeCellInBlock(cellIndexOverlapTop, brlen), UtilFunctions.computeCellInBlock(cellIndexOverlapBottom, brlen), UtilFunctions.computeCellInBlock(cellIndexOverlapLeft, bclen), UtilFunctions.computeCellInBlock(cellIndexOverlapRight, bclen));
    int rowCut = UtilFunctions.computeCellInBlock(ixrange.rowStart, brlen);
    int colCut = UtilFunctions.computeCellInBlock(ixrange.colStart, bclen);
    int rowsInLastBlock = (int) ((ixrange.rowEnd - ixrange.rowStart + 1) % brlen);
    if (rowsInLastBlock == 0)
        rowsInLastBlock = brlen;
    int colsInLastBlock = (int) ((ixrange.colEnd - ixrange.colStart + 1) % bclen);
    if (colsInLastBlock == 0)
        colsInLastBlock = bclen;
    long resultBlockIndexTop = UtilFunctions.computeBlockIndex(cellIndexOverlapTop - ixrange.rowStart + 1, brlen);
    long resultBlockIndexBottom = UtilFunctions.computeBlockIndex(cellIndexOverlapBottom - ixrange.rowStart + 1, brlen);
    long resultBlockIndexLeft = UtilFunctions.computeBlockIndex(cellIndexOverlapLeft - ixrange.colStart + 1, bclen);
    long resultBlockIndexRight = UtilFunctions.computeBlockIndex(cellIndexOverlapRight - ixrange.colStart + 1, bclen);
    int boundaryRlen = brlen;
    int boundaryClen = bclen;
    long finalBlockIndexBottom = UtilFunctions.computeBlockIndex(ixrange.rowEnd - ixrange.rowStart + 1, brlen);
    long finalBlockIndexRight = UtilFunctions.computeBlockIndex(ixrange.colEnd - ixrange.colStart + 1, bclen);
    if (resultBlockIndexBottom == finalBlockIndexBottom)
        boundaryRlen = rowsInLastBlock;
    if (resultBlockIndexRight == finalBlockIndexRight)
        boundaryClen = colsInLastBlock;
    // allocate space for the output value
    for (long r = resultBlockIndexTop; r <= resultBlockIndexBottom; r++) for (long c = resultBlockIndexLeft; c <= resultBlockIndexRight; c++) {
        IndexedMatrixValue out = new IndexedMatrixValue(new MatrixIndexes(), new MatrixBlock());
        out.getIndexes().setIndexes(r, c);
        outlist.add(out);
    }
    // execute actual slice operation
    in.getValue().slice(outlist, tmpRange, rowCut, colCut, brlen, bclen, boundaryRlen, boundaryClen);
}