Examples with IJV org.apache.sysml.runtime.matrix.data.IJV used on opensource projects

Example 1 with IJV

use of org.apache.sysml.runtime.matrix.data.IJV in project incubator-systemml by apache.

the class SpoofOuterProduct method executeCompressed.

private void executeCompressed(CompressedMatrixBlock a, double[] u, double[] v, double[][] b, double[] scalars, double[] c, int m, int n, int k, OutProdType type, int rl, int ru, int cl, int cu) {
    boolean left = (_outerProductType == OutProdType.LEFT_OUTER_PRODUCT);
    Iterator<IJV> iter = !left ? a.getIterator(rl, ru, false) : //cl/cu -> colgroups
    a.getIterator(rl, ru, cl, cu, false);
    while (iter.hasNext()) {
        IJV cell = iter.next();
        int uix = cell.getI() * k;
        int vix = cell.getJ() * k;
        genexecDense(cell.getV(), u, uix, v, vix, b, scalars, c, left ? vix : uix, m, n, k, cell.getI(), cell.getJ());
    }
}

Example 2 with IJV

use of org.apache.sysml.runtime.matrix.data.IJV in project incubator-systemml by apache.

the class SpoofOuterProduct method executeCellwiseCompressed.

private void executeCellwiseCompressed(CompressedMatrixBlock a, double[] u, double[] v, double[][] b, double[] scalars, MatrixBlock out, int m, int n, int k, OutProdType type, int rl, int ru, int cl, int cu) {
    double[] c = out.getDenseBlock();
    SparseBlock csblock = out.getSparseBlock();
    Iterator<IJV> iter = a.getIterator(rl, ru, false);
    while (iter.hasNext()) {
        IJV cell = iter.next();
        int uix = cell.getI() * k;
        int vix = cell.getJ() * k;
        if (type == OutProdType.CELLWISE_OUTER_PRODUCT) {
            if (out.isInSparseFormat()) {
                csblock.allocate(cell.getI());
                csblock.append(cell.getI(), cell.getJ(), genexecCellwise(cell.getV(), u, uix, v, vix, b, scalars, m, n, k, cell.getI(), cell.getJ()));
            } else {
                c[cell.getI() * n + cell.getJ()] = genexecCellwise(cell.getV(), u, uix, v, vix, b, scalars, m, n, k, cell.getI(), cell.getJ());
            }
        } else {
            c[0] += genexecCellwise(cell.getV(), u, uix, v, vix, b, scalars, m, n, k, cell.getI(), cell.getJ());
        }
    }
}

Example 3 with IJV

use of org.apache.sysml.runtime.matrix.data.IJV in project incubator-systemml by apache.

the class ResultMergeLocalFile method createBinaryCellResultFile.

@SuppressWarnings("deprecation")
private void createBinaryCellResultFile(String fnameStaging, String fnameStagingCompare, String fnameNew, MatrixFormatMetaData metadata, boolean withCompare) throws IOException, DMLRuntimeException {
    JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
    Path path = new Path(fnameNew);
    FileSystem fs = IOUtilFunctions.getFileSystem(path, job);
    MatrixCharacteristics mc = metadata.getMatrixCharacteristics();
    long rlen = mc.getRows();
    long clen = mc.getCols();
    int brlen = mc.getRowsPerBlock();
    int bclen = mc.getColsPerBlock();
    MatrixIndexes indexes = new MatrixIndexes(1, 1);
    MatrixCell cell = new MatrixCell(0);
    //beware ca 50ms
    SequenceFile.Writer out = new SequenceFile.Writer(fs, job, path, MatrixIndexes.class, MatrixCell.class);
    try {
        boolean written = false;
        for (long brow = 1; brow <= (long) Math.ceil(rlen / (double) brlen); brow++) for (long bcol = 1; bcol <= (long) Math.ceil(clen / (double) bclen); bcol++) {
            File dir = new File(fnameStaging + "/" + brow + "_" + bcol);
            File dir2 = new File(fnameStagingCompare + "/" + brow + "_" + bcol);
            MatrixBlock mb = null;
            long row_offset = (brow - 1) * brlen + 1;
            long col_offset = (bcol - 1) * bclen + 1;
            if (dir.exists()) {
                if (//WITH COMPARE BLOCK
                withCompare && dir2.exists()) {
                    //copy only values that are different from the original
                    String[] lnames2 = dir2.list();
                    if (//there should be exactly 1 compare block
                    lnames2.length != 1)
                        throw new DMLRuntimeException("Unable to merge results because multiple compare blocks found.");
                    mb = StagingFileUtils.readCellList2BlockFromLocal(dir2 + "/" + lnames2[0], brlen, bclen);
                    boolean appendOnly = mb.isInSparseFormat();
                    double[][] compare = DataConverter.convertToDoubleMatrix(mb);
                    String[] lnames = dir.list();
                    for (String lname : lnames) {
                        MatrixBlock tmp = StagingFileUtils.readCellList2BlockFromLocal(dir + "/" + lname, brlen, bclen);
                        mergeWithComp(mb, tmp, compare);
                    }
                    //sort sparse due to append-only
                    if (appendOnly)
                        mb.sortSparseRows();
                    //change sparsity if required after 
                    mb.examSparsity();
                } else //WITHOUT COMPARE BLOCK
                {
                    //copy all non-zeros from all workers
                    String[] lnames = dir.list();
                    boolean appendOnly = false;
                    for (String lname : lnames) {
                        if (mb == null) {
                            mb = StagingFileUtils.readCellList2BlockFromLocal(dir + "/" + lname, brlen, bclen);
                            appendOnly = mb.isInSparseFormat();
                        } else {
                            MatrixBlock tmp = StagingFileUtils.readCellList2BlockFromLocal(dir + "/" + lname, brlen, bclen);
                            mergeWithoutComp(mb, tmp, appendOnly);
                        }
                    }
                    //sort sparse due to append-only
                    if (appendOnly)
                        mb.sortSparseRows();
                    //change sparsity if required after 
                    mb.examSparsity();
                }
            }
            //write the block to binary cell
            if (mb != null) {
                if (mb.isInSparseFormat()) {
                    Iterator<IJV> iter = mb.getSparseBlockIterator();
                    while (iter.hasNext()) {
                        IJV lcell = iter.next();
                        indexes.setIndexes(row_offset + lcell.getI(), col_offset + lcell.getJ());
                        cell.setValue(lcell.getV());
                        out.append(indexes, cell);
                        written = true;
                    }
                } else {
                    for (int i = 0; i < brlen; i++) for (int j = 0; j < bclen; j++) {
                        double lvalue = mb.getValueDenseUnsafe(i, j);
                        if (//for nnz
                        lvalue != 0) {
                            indexes.setIndexes(row_offset + i, col_offset + j);
                            cell.setValue(lvalue);
                            out.append(indexes, cell);
                            written = true;
                        }
                    }
                }
            }
        }
        if (!written)
            out.append(indexes, cell);
    } finally {
        IOUtilFunctions.closeSilently(out);
    }
}

Example 4 with IJV

use of org.apache.sysml.runtime.matrix.data.IJV in project incubator-systemml by apache.

the class MLContextConversionUtil method matrixObjectToListStringIJV.

/**
 * Convert a {@code MatrixObject} to a {@code List<String>} in IJV format.
 *
 * @param matrixObject
 *            the {@code MatrixObject}
 * @return the {@code MatrixObject} converted to a {@code List<String>}
 */
public static List<String> matrixObjectToListStringIJV(MatrixObject matrixObject) {
    MatrixBlock mb = matrixObject.acquireRead();
    int rows = mb.getNumRows();
    int cols = mb.getNumColumns();
    List<String> list = new ArrayList<>();
    if (mb.getNonZeros() > 0) {
        if (mb.isInSparseFormat()) {
            Iterator<IJV> iter = mb.getSparseBlockIterator();
            StringBuilder sb = null;
            while (iter.hasNext()) {
                IJV cell = iter.next();
                sb = new StringBuilder();
                sb.append(cell.getI() + 1);
                sb.append(" ");
                sb.append(cell.getJ() + 1);
                sb.append(" ");
                sb.append(cell.getV());
                list.add(sb.toString());
            }
        } else {
            StringBuilder sb = null;
            for (int i = 0; i < rows; i++) {
                sb = new StringBuilder();
                for (int j = 0; j < cols; j++) {
                    sb = new StringBuilder();
                    sb.append(i + 1);
                    sb.append(" ");
                    sb.append(j + 1);
                    sb.append(" ");
                    sb.append(mb.getValueDenseUnsafe(i, j));
                    list.add(sb.toString());
                }
            }
        }
    }
    matrixObject.release();
    return list;
}

Example 5 with IJV

use of org.apache.sysml.runtime.matrix.data.IJV in project incubator-systemml by apache.

the class ResultMergeLocalFile method createTextCellResultFile.

private void createTextCellResultFile(String fnameStaging, String fnameStagingCompare, String fnameNew, MetaDataFormat metadata, boolean withCompare) throws IOException, DMLRuntimeException {
    JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
    Path path = new Path(fnameNew);
    FileSystem fs = IOUtilFunctions.getFileSystem(path, job);
    MatrixCharacteristics mc = metadata.getMatrixCharacteristics();
    long rlen = mc.getRows();
    long clen = mc.getCols();
    int brlen = mc.getRowsPerBlock();
    int bclen = mc.getColsPerBlock();
    try (BufferedWriter out = new BufferedWriter(new OutputStreamWriter(fs.create(path, true)))) {
        // for obj reuse and preventing repeated buffer re-allocations
        StringBuilder sb = new StringBuilder();
        boolean written = false;
        for (long brow = 1; brow <= (long) Math.ceil(rlen / (double) brlen); brow++) for (long bcol = 1; bcol <= (long) Math.ceil(clen / (double) bclen); bcol++) {
            File dir = new File(fnameStaging + "/" + brow + "_" + bcol);
            File dir2 = new File(fnameStagingCompare + "/" + brow + "_" + bcol);
            MatrixBlock mb = null;
            long row_offset = (brow - 1) * brlen + 1;
            long col_offset = (bcol - 1) * bclen + 1;
            if (dir.exists()) {
                if (// WITH COMPARE BLOCK
                withCompare && dir2.exists()) {
                    // copy only values that are different from the original
                    String[] lnames2 = dir2.list();
                    if (// there should be exactly 1 compare block
                    lnames2.length != 1)
                        throw new DMLRuntimeException("Unable to merge results because multiple compare blocks found.");
                    mb = StagingFileUtils.readCellList2BlockFromLocal(dir2 + "/" + lnames2[0], brlen, bclen);
                    boolean appendOnly = mb.isInSparseFormat();
                    DenseBlock compare = DataConverter.convertToDenseBlock(mb, false);
                    for (String lname : dir.list()) {
                        MatrixBlock tmp = StagingFileUtils.readCellList2BlockFromLocal(dir + "/" + lname, brlen, bclen);
                        mergeWithComp(mb, tmp, compare);
                    }
                    // sort sparse and exam sparsity due to append-only
                    if (appendOnly && !_isAccum)
                        mb.sortSparseRows();
                    // change sparsity if required after
                    mb.examSparsity();
                } else // WITHOUT COMPARE BLOCK
                {
                    // copy all non-zeros from all workers
                    boolean appendOnly = false;
                    for (String lname : dir.list()) {
                        if (mb == null) {
                            mb = StagingFileUtils.readCellList2BlockFromLocal(dir + "/" + lname, brlen, bclen);
                            appendOnly = mb.isInSparseFormat();
                        } else {
                            MatrixBlock tmp = StagingFileUtils.readCellList2BlockFromLocal(dir + "/" + lname, brlen, bclen);
                            mergeWithoutComp(mb, tmp, appendOnly);
                        }
                    }
                    // sort sparse due to append-only
                    if (appendOnly && !_isAccum)
                        mb.sortSparseRows();
                    // change sparsity if required after
                    mb.examSparsity();
                }
            }
            // write the block to text cell
            if (mb != null) {
                if (mb.isInSparseFormat()) {
                    Iterator<IJV> iter = mb.getSparseBlockIterator();
                    while (iter.hasNext()) {
                        IJV lcell = iter.next();
                        sb.append(row_offset + lcell.getI());
                        sb.append(' ');
                        sb.append(col_offset + lcell.getJ());
                        sb.append(' ');
                        sb.append(lcell.getV());
                        sb.append('\n');
                        out.write(sb.toString());
                        sb.setLength(0);
                        written = true;
                    }
                } else {
                    for (int i = 0; i < brlen; i++) for (int j = 0; j < bclen; j++) {
                        double lvalue = mb.getValueDenseUnsafe(i, j);
                        if (// for nnz
                        lvalue != 0) {
                            sb.append(row_offset + i);
                            sb.append(' ');
                            sb.append(col_offset + j);
                            sb.append(' ');
                            sb.append(lvalue);
                            sb.append('\n');
                            out.write(sb.toString());
                            sb.setLength(0);
                            written = true;
                        }
                    }
                }
            }
        }
        if (!written)
            out.write(IOUtilFunctions.EMPTY_TEXT_LINE);
    }
}