Examples with IJV org.apache.sysml.runtime.matrix.data.IJV used on opensource projects

Example 41 with IJV

use of org.apache.sysml.runtime.matrix.data.IJV in project incubator-systemml by apache.

the class DataConverter method convertToDoubleVector.

public static double[] convertToDoubleVector(MatrixBlock mb) {
    int rows = mb.getNumRows();
    int cols = mb.getNumColumns();
    //0-initialized 
    double[] ret = new double[rows * cols];
    if (mb.getNonZeros() > 0) {
        if (mb.isInSparseFormat()) {
            Iterator<IJV> iter = mb.getSparseBlockIterator();
            while (iter.hasNext()) {
                IJV cell = iter.next();
                ret[cell.getI() * cols + cell.getJ()] = cell.getV();
            }
        } else {
            //memcopy row major representation if at least 1 non-zero
            System.arraycopy(mb.getDenseBlock(), 0, ret, 0, rows * cols);
        }
    }
    return ret;
}

Example 42 with IJV

use of org.apache.sysml.runtime.matrix.data.IJV in project incubator-systemml by apache.

the class RowClassMeet method getRow.

double[] getRow(MatrixBlock B, double[] bRow, int i) {
    if (B.getNumRows() == 1)
        i = 0;
    Arrays.fill(bRow, 0);
    if (B.isInSparseFormat()) {
        Iterator<IJV> iter = B.getSparseBlockIterator(i, i + 1);
        while (iter.hasNext()) {
            IJV ijv = iter.next();
            bRow[ijv.getJ()] = ijv.getV();
        }
    } else {
        double[] denseBlk = B.getDenseBlock();
        if (denseBlk != null)
            System.arraycopy(denseBlk, i * B.getNumColumns(), bRow, 0, B.getNumColumns());
    }
    return bRow;
}

Example 43 with IJV

use of org.apache.sysml.runtime.matrix.data.IJV in project incubator-systemml by apache.

the class ResultMergeLocalFile method createTextCellResultFile.

private void createTextCellResultFile(String fnameStaging, String fnameStagingCompare, String fnameNew, MatrixFormatMetaData metadata, boolean withCompare) throws IOException, DMLRuntimeException {
    JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
    Path path = new Path(fnameNew);
    FileSystem fs = IOUtilFunctions.getFileSystem(path, job);
    MatrixCharacteristics mc = metadata.getMatrixCharacteristics();
    long rlen = mc.getRows();
    long clen = mc.getCols();
    int brlen = mc.getRowsPerBlock();
    int bclen = mc.getColsPerBlock();
    BufferedWriter out = new BufferedWriter(new OutputStreamWriter(fs.create(path, true)));
    try {
        //for obj reuse and preventing repeated buffer re-allocations
        StringBuilder sb = new StringBuilder();
        boolean written = false;
        for (long brow = 1; brow <= (long) Math.ceil(rlen / (double) brlen); brow++) for (long bcol = 1; bcol <= (long) Math.ceil(clen / (double) bclen); bcol++) {
            File dir = new File(fnameStaging + "/" + brow + "_" + bcol);
            File dir2 = new File(fnameStagingCompare + "/" + brow + "_" + bcol);
            MatrixBlock mb = null;
            long row_offset = (brow - 1) * brlen + 1;
            long col_offset = (bcol - 1) * bclen + 1;
            if (dir.exists()) {
                if (//WITH COMPARE BLOCK
                withCompare && dir2.exists()) {
                    //copy only values that are different from the original
                    String[] lnames2 = dir2.list();
                    if (//there should be exactly 1 compare block
                    lnames2.length != 1)
                        throw new DMLRuntimeException("Unable to merge results because multiple compare blocks found.");
                    mb = StagingFileUtils.readCellList2BlockFromLocal(dir2 + "/" + lnames2[0], brlen, bclen);
                    boolean appendOnly = mb.isInSparseFormat();
                    double[][] compare = DataConverter.convertToDoubleMatrix(mb);
                    String[] lnames = dir.list();
                    for (String lname : lnames) {
                        MatrixBlock tmp = StagingFileUtils.readCellList2BlockFromLocal(dir + "/" + lname, brlen, bclen);
                        mergeWithComp(mb, tmp, compare);
                    }
                    //sort sparse and exam sparsity due to append-only
                    if (appendOnly)
                        mb.sortSparseRows();
                    //change sparsity if required after 
                    mb.examSparsity();
                } else //WITHOUT COMPARE BLOCK
                {
                    //copy all non-zeros from all workers
                    String[] lnames = dir.list();
                    boolean appendOnly = false;
                    for (String lname : lnames) {
                        if (mb == null) {
                            mb = StagingFileUtils.readCellList2BlockFromLocal(dir + "/" + lname, brlen, bclen);
                            appendOnly = mb.isInSparseFormat();
                        } else {
                            MatrixBlock tmp = StagingFileUtils.readCellList2BlockFromLocal(dir + "/" + lname, brlen, bclen);
                            mergeWithoutComp(mb, tmp, appendOnly);
                        }
                    }
                    //sort sparse due to append-only
                    if (appendOnly)
                        mb.sortSparseRows();
                    //change sparsity if required after 
                    mb.examSparsity();
                }
            }
            //write the block to text cell
            if (mb != null) {
                if (mb.isInSparseFormat()) {
                    Iterator<IJV> iter = mb.getSparseBlockIterator();
                    while (iter.hasNext()) {
                        IJV lcell = iter.next();
                        sb.append(row_offset + lcell.getI());
                        sb.append(' ');
                        sb.append(col_offset + lcell.getJ());
                        sb.append(' ');
                        sb.append(lcell.getV());
                        sb.append('\n');
                        out.write(sb.toString());
                        sb.setLength(0);
                        written = true;
                    }
                } else {
                    for (int i = 0; i < brlen; i++) for (int j = 0; j < bclen; j++) {
                        double lvalue = mb.getValueDenseUnsafe(i, j);
                        if (//for nnz
                        lvalue != 0) {
                            sb.append(row_offset + i);
                            sb.append(' ');
                            sb.append(col_offset + j);
                            sb.append(' ');
                            sb.append(lvalue);
                            sb.append('\n');
                            out.write(sb.toString());
                            sb.setLength(0);
                            written = true;
                        }
                    }
                }
            }
        }
        if (!written)
            out.write("1 1 0\n");
    } finally {
        IOUtilFunctions.closeSilently(out);
    }
}