Examples with IJV org.apache.sysml.runtime.matrix.data.IJV used on opensource projects

Example 21 with IJV

use of org.apache.sysml.runtime.matrix.data.IJV in project incubator-systemml by apache.

the class SparseBlockIndexRange method runSparseBlockIndexRangeTest.

/**
 * @param btype
 * @param sparsity
 */
private void runSparseBlockIndexRangeTest(SparseBlock.Type btype, double sparsity, UpdateType utype) {
    try {
        // data generation
        double[][] A = getRandomMatrix(rows, cols, -10, 10, sparsity, 456);
        // init sparse block
        SparseBlock sblock = null;
        MatrixBlock mbtmp = DataConverter.convertToMatrixBlock(A);
        SparseBlock srtmp = mbtmp.getSparseBlock();
        switch(btype) {
            case MCSR:
                sblock = new SparseBlockMCSR(srtmp);
                break;
            case CSR:
                sblock = new SparseBlockCSR(srtmp);
                break;
            case COO:
                sblock = new SparseBlockCOO(srtmp);
                break;
        }
        // delete range per row via set
        if (utype == UpdateType.DELETE) {
            for (int i = 0; i < rows; i++) {
                sblock.deleteIndexRange(i, cl, cu);
                Arrays.fill(A[i], cl, cu, 0);
            }
        } else if (utype == UpdateType.INSERT) {
            double[] vals = new double[cu - cl];
            for (int j = cl; j < cu; j++) vals[j - cl] = j;
            for (int i = 0; i < rows; i++) {
                sblock.setIndexRange(i, cl, cu, vals, 0, cu - cl);
                System.arraycopy(vals, 0, A[i], cl, cu - cl);
            }
        }
        // check for correct number of non-zeros
        int[] rnnz = new int[rows];
        int nnz = 0;
        for (int i = 0; i < rows; i++) {
            for (int j = 0; j < cols; j++) rnnz[i] += (A[i][j] != 0) ? 1 : 0;
            nnz += rnnz[i];
        }
        if (nnz != sblock.size())
            Assert.fail("Wrong number of non-zeros: " + sblock.size() + ", expected: " + nnz);
        // check correct isEmpty return
        for (int i = 0; i < rows; i++) if (sblock.isEmpty(i) != (rnnz[i] == 0))
            Assert.fail("Wrong isEmpty(row) result for row nnz: " + rnnz[i]);
        // check correct values
        Iterator<IJV> iter = sblock.getIterator();
        int count = 0;
        while (iter.hasNext()) {
            IJV cell = iter.next();
            if (cell.getV() != A[cell.getI()][cell.getJ()])
                Assert.fail("Wrong value returned by iterator: " + cell.getV() + ", expected: " + A[cell.getI()][cell.getJ()]);
            count++;
        }
        if (count != nnz)
            Assert.fail("Wrong number of values returned by iterator: " + count + ", expected: " + nnz);
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new RuntimeException(ex);
    }
}

Example 22 with IJV

use of org.apache.sysml.runtime.matrix.data.IJV in project incubator-systemml by apache.

the class MLContextConversionUtil method matrixObjectToListStringCSV.

/**
 * Convert a {@code MatrixObject} to a {@code List<String>} in CSV format.
 *
 * @param matrixObject
 *            the {@code MatrixObject}
 * @return the {@code MatrixObject} converted to a {@code List<String>}
 */
public static List<String> matrixObjectToListStringCSV(MatrixObject matrixObject) {
    MatrixBlock mb = matrixObject.acquireRead();
    int rows = mb.getNumRows();
    int cols = mb.getNumColumns();
    List<String> list = new ArrayList<>();
    if (!mb.isEmptyBlock(false)) {
        if (mb.isInSparseFormat()) {
            Iterator<IJV> iter = mb.getSparseBlockIterator();
            int prevCellRow = -1;
            StringBuilder sb = null;
            while (iter.hasNext()) {
                IJV cell = iter.next();
                int i = cell.getI();
                double v = cell.getV();
                if (i > prevCellRow) {
                    if (sb == null) {
                        sb = new StringBuilder();
                    } else {
                        list.add(sb.toString());
                        sb = new StringBuilder();
                    }
                    sb.append(v);
                    prevCellRow = i;
                } else if (i == prevCellRow) {
                    sb.append(",");
                    sb.append(v);
                }
            }
            if (sb != null) {
                list.add(sb.toString());
            }
        } else {
            for (int i = 0; i < rows; i++) {
                StringBuilder sb = new StringBuilder();
                for (int j = 0; j < cols; j++) {
                    if (j > 0) {
                        sb.append(",");
                    }
                    sb.append(mb.getValueDenseUnsafe(i, j));
                }
                list.add(sb.toString());
            }
        }
    }
    matrixObject.release();
    return list;
}

Example 23 with IJV

use of org.apache.sysml.runtime.matrix.data.IJV in project incubator-systemml by apache.

the class SparseBlockIterator method runSparseBlockIteratorTest.

/**
 * @param sparseM1
 * @param sparseM2
 * @param instType
 */
private void runSparseBlockIteratorTest(SparseBlock.Type btype, double sparsity, boolean partial) {
    try {
        // data generation
        double[][] A = getRandomMatrix(rows, cols, -10, 10, sparsity, 8765432);
        // init sparse block
        SparseBlock sblock = null;
        MatrixBlock mbtmp = DataConverter.convertToMatrixBlock(A);
        SparseBlock srtmp = mbtmp.getSparseBlock();
        switch(btype) {
            case MCSR:
                sblock = new SparseBlockMCSR(srtmp);
                break;
            case CSR:
                sblock = new SparseBlockCSR(srtmp);
                break;
            case COO:
                sblock = new SparseBlockCOO(srtmp);
                break;
        }
        // check for correct number of non-zeros
        int[] rnnz = new int[rows];
        int nnz = 0;
        int rl = partial ? rlPartial : 0;
        for (int i = rl; i < rows; i++) {
            for (int j = 0; j < cols; j++) rnnz[i] += (A[i][j] != 0) ? 1 : 0;
            nnz += rnnz[i];
        }
        if (!partial && nnz != sblock.size())
            Assert.fail("Wrong number of non-zeros: " + sblock.size() + ", expected: " + nnz);
        // check correct isEmpty return
        for (int i = rl; i < rows; i++) if (sblock.isEmpty(i) != (rnnz[i] == 0))
            Assert.fail("Wrong isEmpty(row) result for row nnz: " + rnnz[i]);
        // check correct values
        Iterator<IJV> iter = !partial ? sblock.getIterator() : sblock.getIterator(rl, rows);
        int count = 0;
        while (iter.hasNext()) {
            IJV cell = iter.next();
            if (cell.getV() != A[cell.getI()][cell.getJ()])
                Assert.fail("Wrong value returned by iterator: " + cell.getV() + ", expected: " + A[cell.getI()][cell.getJ()]);
            count++;
        }
        if (count != nnz)
            Assert.fail("Wrong number of values returned by iterator: " + count + ", expected: " + nnz);
    } catch (Exception ex) {
        ex.printStackTrace();
        throw new RuntimeException(ex);
    }
}

Example 24 with IJV

use of org.apache.sysml.runtime.matrix.data.IJV in project incubator-systemml by apache.

the class DataPartitionerLocal method partitionBinaryBlock2BinaryCell.

@SuppressWarnings("deprecation")
private void partitionBinaryBlock2BinaryCell(String fname, String fnameStaging, String fnameNew, long rlen, long clen, int brlen, int bclen) {
    try {
        // STEP 1: read matrix from HDFS and write blocks to local staging area
        // check and add input path
        JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
        Path path = new Path(fname);
        FileSystem fs = IOUtilFunctions.getFileSystem(path, job);
        // prepare sequence file reader, and write to local staging area
        MatrixIndexes key = new MatrixIndexes();
        MatrixBlock value = new MatrixBlock();
        LinkedList<Cell> buffer = new LinkedList<>();
        for (Path lpath : IOUtilFunctions.getSequenceFilePaths(fs, path)) {
            SequenceFile.Reader reader = new SequenceFile.Reader(fs, lpath, job);
            try {
                while (// for each block
                reader.next(key, value)) {
                    long row_offset = (key.getRowIndex() - 1) * brlen;
                    long col_offset = (key.getColumnIndex() - 1) * bclen;
                    long rows = value.getNumRows();
                    long cols = value.getNumColumns();
                    // bound check per block
                    if (row_offset + rows < 1 || row_offset + rows > rlen || col_offset + cols < 1 || col_offset + cols > clen) {
                        throw new IOException("Matrix block [" + (row_offset + 1) + ":" + (row_offset + rows) + "," + (col_offset + 1) + ":" + (col_offset + cols) + "] " + "out of overall matrix range [1:" + rlen + ",1:" + clen + "].");
                    }
                    boolean sparse = value.isInSparseFormat();
                    if (// SPARSE
                    sparse) {
                        Iterator<IJV> iter = value.getSparseBlockIterator();
                        while (iter.hasNext()) {
                            IJV lcell = iter.next();
                            Cell tmp = new Cell(row_offset + lcell.getI() + 1, col_offset + lcell.getJ() + 1, lcell.getV());
                            buffer.addLast(tmp);
                        }
                    } else // DENSE
                    {
                        for (int i = 0; i < rows; i++) for (int j = 0; j < cols; j++) {
                            double lvalue = value.getValueDenseUnsafe(i, j);
                            if (// for nnz
                            lvalue != 0) {
                                Cell tmp = new Cell(row_offset + i + 1, col_offset + j + 1, lvalue);
                                buffer.addLast(tmp);
                            }
                        }
                    }
                    appendCellBufferToStagingArea(fnameStaging, buffer, brlen, bclen);
                    buffer.clear();
                }
            } finally {
                IOUtilFunctions.closeSilently(reader);
            }
        }
        // STEP 2: read matrix blocks from staging area and write matrix to HDFS
        String[] fnamesPartitions = new File(fnameStaging).list();
        if (PARALLEL) {
            int len = Math.min(fnamesPartitions.length, _par);
            Thread[] threads = new Thread[len];
            for (int i = 0; i < len; i++) {
                int start = i * (int) Math.ceil(((double) fnamesPartitions.length) / len);
                int end = (i + 1) * (int) Math.ceil(((double) fnamesPartitions.length) / len) - 1;
                end = Math.min(end, fnamesPartitions.length - 1);
                threads[i] = new Thread(new DataPartitionerWorkerBinaryCell(job, fnameNew, fnameStaging, fnamesPartitions, start, end));
                threads[i].start();
            }
            for (Thread t : threads) t.join();
        } else {
            for (String pdir : fnamesPartitions) writeBinaryCellSequenceFileToHDFS(job, fnameNew, fnameStaging + "/" + pdir);
        }
    } catch (Exception e) {
        throw new DMLRuntimeException("Unable to partition binary block matrix.", e);
    }
}

Example 25 with IJV

use of org.apache.sysml.runtime.matrix.data.IJV in project incubator-systemml by apache.

the class SpoofCellwise method executeCompressedNoAgg.

private long executeCompressedNoAgg(CompressedMatrixBlock a, SideInput[] b, double[] scalars, MatrixBlock out, int m, int n, boolean sparseSafe, int rl, int ru) {
    double[] c = (out.getDenseBlock() != null) ? out.getDenseBlockValues() : null;
    SparseBlock csblock = out.getSparseBlock();
    // whenever k/2 * BITMAP_BLOCK_SZ > m (i.e., it does not limit parallelism)
    if (out.isInSparseFormat() && rl % BitmapEncoder.BITMAP_BLOCK_SZ == 0 && ru % BitmapEncoder.BITMAP_BLOCK_SZ == 0) {
        int[] rnnz = a.countNonZerosPerRow(rl, ru);
        for (int i = rl; i < ru; i++) csblock.allocate(i, rnnz[i - rl]);
    }
    long lnnz = 0;
    Iterator<IJV> iter = a.getIterator(rl, ru, !sparseSafe);
    while (iter.hasNext()) {
        IJV cell = iter.next();
        double val = genexec(cell.getV(), b, scalars, m, n, cell.getI(), cell.getJ());
        if (out.isInSparseFormat()) {
            csblock.allocate(cell.getI());
            csblock.append(cell.getI(), cell.getJ(), val);
        } else
            c[cell.getI() * n + cell.getJ()] = val;
        lnnz += (val != 0) ? 1 : 0;
    }
    return lnnz;
}