Examples with DenseBlock org.apache.sysml.runtime.matrix.data.DenseBlock used on opensource projects

Example 1 with DenseBlock

use of org.apache.sysml.runtime.matrix.data.DenseBlock in project incubator-systemml by apache.

the class ResultMergeLocalFile method createTextCellResultFile.

private void createTextCellResultFile(String fnameStaging, String fnameStagingCompare, String fnameNew, MetaDataFormat metadata, boolean withCompare) throws IOException, DMLRuntimeException {
    JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
    Path path = new Path(fnameNew);
    FileSystem fs = IOUtilFunctions.getFileSystem(path, job);
    MatrixCharacteristics mc = metadata.getMatrixCharacteristics();
    long rlen = mc.getRows();
    long clen = mc.getCols();
    int brlen = mc.getRowsPerBlock();
    int bclen = mc.getColsPerBlock();
    try (BufferedWriter out = new BufferedWriter(new OutputStreamWriter(fs.create(path, true)))) {
        // for obj reuse and preventing repeated buffer re-allocations
        StringBuilder sb = new StringBuilder();
        boolean written = false;
        for (long brow = 1; brow <= (long) Math.ceil(rlen / (double) brlen); brow++) for (long bcol = 1; bcol <= (long) Math.ceil(clen / (double) bclen); bcol++) {
            File dir = new File(fnameStaging + "/" + brow + "_" + bcol);
            File dir2 = new File(fnameStagingCompare + "/" + brow + "_" + bcol);
            MatrixBlock mb = null;
            long row_offset = (brow - 1) * brlen + 1;
            long col_offset = (bcol - 1) * bclen + 1;
            if (dir.exists()) {
                if (// WITH COMPARE BLOCK
                withCompare && dir2.exists()) {
                    // copy only values that are different from the original
                    String[] lnames2 = dir2.list();
                    if (// there should be exactly 1 compare block
                    lnames2.length != 1)
                        throw new DMLRuntimeException("Unable to merge results because multiple compare blocks found.");
                    mb = StagingFileUtils.readCellList2BlockFromLocal(dir2 + "/" + lnames2[0], brlen, bclen);
                    boolean appendOnly = mb.isInSparseFormat();
                    DenseBlock compare = DataConverter.convertToDenseBlock(mb, false);
                    for (String lname : dir.list()) {
                        MatrixBlock tmp = StagingFileUtils.readCellList2BlockFromLocal(dir + "/" + lname, brlen, bclen);
                        mergeWithComp(mb, tmp, compare);
                    }
                    // sort sparse and exam sparsity due to append-only
                    if (appendOnly && !_isAccum)
                        mb.sortSparseRows();
                    // change sparsity if required after
                    mb.examSparsity();
                } else // WITHOUT COMPARE BLOCK
                {
                    // copy all non-zeros from all workers
                    boolean appendOnly = false;
                    for (String lname : dir.list()) {
                        if (mb == null) {
                            mb = StagingFileUtils.readCellList2BlockFromLocal(dir + "/" + lname, brlen, bclen);
                            appendOnly = mb.isInSparseFormat();
                        } else {
                            MatrixBlock tmp = StagingFileUtils.readCellList2BlockFromLocal(dir + "/" + lname, brlen, bclen);
                            mergeWithoutComp(mb, tmp, appendOnly);
                        }
                    }
                    // sort sparse due to append-only
                    if (appendOnly && !_isAccum)
                        mb.sortSparseRows();
                    // change sparsity if required after
                    mb.examSparsity();
                }
            }
            // write the block to text cell
            if (mb != null) {
                if (mb.isInSparseFormat()) {
                    Iterator<IJV> iter = mb.getSparseBlockIterator();
                    while (iter.hasNext()) {
                        IJV lcell = iter.next();
                        sb.append(row_offset + lcell.getI());
                        sb.append(' ');
                        sb.append(col_offset + lcell.getJ());
                        sb.append(' ');
                        sb.append(lcell.getV());
                        sb.append('\n');
                        out.write(sb.toString());
                        sb.setLength(0);
                        written = true;
                    }
                } else {
                    for (int i = 0; i < brlen; i++) for (int j = 0; j < bclen; j++) {
                        double lvalue = mb.getValueDenseUnsafe(i, j);
                        if (// for nnz
                        lvalue != 0) {
                            sb.append(row_offset + i);
                            sb.append(' ');
                            sb.append(col_offset + j);
                            sb.append(' ');
                            sb.append(lvalue);
                            sb.append('\n');
                            out.write(sb.toString());
                            sb.setLength(0);
                            written = true;
                        }
                    }
                }
            }
        }
        if (!written)
            out.write(IOUtilFunctions.EMPTY_TEXT_LINE);
    }
}

Example 2 with DenseBlock

use of org.apache.sysml.runtime.matrix.data.DenseBlock in project incubator-systemml by apache.

the class ResultMergeLocalFile method createBinaryBlockResultFile.

@SuppressWarnings("deprecation")
private void createBinaryBlockResultFile(String fnameStaging, String fnameStagingCompare, String fnameNew, MetaDataFormat metadata, boolean withCompare) throws IOException, DMLRuntimeException {
    JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
    Path path = new Path(fnameNew);
    FileSystem fs = IOUtilFunctions.getFileSystem(path, job);
    MatrixCharacteristics mc = metadata.getMatrixCharacteristics();
    long rlen = mc.getRows();
    long clen = mc.getCols();
    int brlen = mc.getRowsPerBlock();
    int bclen = mc.getColsPerBlock();
    // beware ca 50ms
    SequenceFile.Writer writer = new SequenceFile.Writer(fs, job, path, MatrixIndexes.class, MatrixBlock.class);
    try {
        MatrixIndexes indexes = new MatrixIndexes();
        for (long brow = 1; brow <= (long) Math.ceil(rlen / (double) brlen); brow++) for (long bcol = 1; bcol <= (long) Math.ceil(clen / (double) bclen); bcol++) {
            File dir = new File(fnameStaging + "/" + brow + "_" + bcol);
            File dir2 = new File(fnameStagingCompare + "/" + brow + "_" + bcol);
            MatrixBlock mb = null;
            if (dir.exists()) {
                if (// WITH COMPARE BLOCK
                withCompare && dir2.exists()) {
                    // copy only values that are different from the original
                    String[] lnames2 = dir2.list();
                    if (// there should be exactly 1 compare block
                    lnames2.length != 1)
                        throw new DMLRuntimeException("Unable to merge results because multiple compare blocks found.");
                    mb = LocalFileUtils.readMatrixBlockFromLocal(dir2 + "/" + lnames2[0]);
                    boolean appendOnly = mb.isInSparseFormat();
                    DenseBlock compare = DataConverter.convertToDenseBlock(mb, false);
                    for (String lname : dir.list()) {
                        MatrixBlock tmp = LocalFileUtils.readMatrixBlockFromLocal(dir + "/" + lname);
                        mergeWithComp(mb, tmp, compare);
                    }
                    // sort sparse due to append-only
                    if (appendOnly && !_isAccum)
                        mb.sortSparseRows();
                    // change sparsity if required after
                    mb.examSparsity();
                } else // WITHOUT COMPARE BLOCK
                {
                    // copy all non-zeros from all workers
                    boolean appendOnly = false;
                    for (String lname : dir.list()) {
                        if (mb == null) {
                            mb = LocalFileUtils.readMatrixBlockFromLocal(dir + "/" + lname);
                            appendOnly = mb.isInSparseFormat();
                        } else {
                            MatrixBlock tmp = LocalFileUtils.readMatrixBlockFromLocal(dir + "/" + lname);
                            mergeWithoutComp(mb, tmp, appendOnly);
                        }
                    }
                    // sort sparse due to append-only
                    if (appendOnly && !_isAccum)
                        mb.sortSparseRows();
                    // change sparsity if required after
                    mb.examSparsity();
                }
            } else {
                // NOTE: whenever runtime does not need all blocks anymore, this can be removed
                int maxRow = (int) (((brow - 1) * brlen + brlen < rlen) ? brlen : rlen - (brow - 1) * brlen);
                int maxCol = (int) (((bcol - 1) * bclen + bclen < clen) ? bclen : clen - (bcol - 1) * bclen);
                mb = new MatrixBlock(maxRow, maxCol, true);
            }
            // mb.examSparsity(); //done on write anyway and mb not reused
            indexes.setIndexes(brow, bcol);
            writer.append(indexes, mb);
        }
    } finally {
        IOUtilFunctions.closeSilently(writer);
    }
}

Example 3 with DenseBlock

use of org.apache.sysml.runtime.matrix.data.DenseBlock in project incubator-systemml by apache.

the class ResultMergeLocalFile method createBinaryCellResultFile.

@SuppressWarnings("deprecation")
private void createBinaryCellResultFile(String fnameStaging, String fnameStagingCompare, String fnameNew, MetaDataFormat metadata, boolean withCompare) throws IOException, DMLRuntimeException {
    JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
    Path path = new Path(fnameNew);
    FileSystem fs = IOUtilFunctions.getFileSystem(path, job);
    MatrixCharacteristics mc = metadata.getMatrixCharacteristics();
    long rlen = mc.getRows();
    long clen = mc.getCols();
    int brlen = mc.getRowsPerBlock();
    int bclen = mc.getColsPerBlock();
    MatrixIndexes indexes = new MatrixIndexes(1, 1);
    MatrixCell cell = new MatrixCell(0);
    // beware ca 50ms
    SequenceFile.Writer out = new SequenceFile.Writer(fs, job, path, MatrixIndexes.class, MatrixCell.class);
    try {
        boolean written = false;
        for (long brow = 1; brow <= (long) Math.ceil(rlen / (double) brlen); brow++) for (long bcol = 1; bcol <= (long) Math.ceil(clen / (double) bclen); bcol++) {
            File dir = new File(fnameStaging + "/" + brow + "_" + bcol);
            File dir2 = new File(fnameStagingCompare + "/" + brow + "_" + bcol);
            MatrixBlock mb = null;
            long row_offset = (brow - 1) * brlen + 1;
            long col_offset = (bcol - 1) * bclen + 1;
            if (dir.exists()) {
                if (// WITH COMPARE BLOCK
                withCompare && dir2.exists()) {
                    // copy only values that are different from the original
                    String[] lnames2 = dir2.list();
                    if (// there should be exactly 1 compare block
                    lnames2.length != 1)
                        throw new DMLRuntimeException("Unable to merge results because multiple compare blocks found.");
                    mb = StagingFileUtils.readCellList2BlockFromLocal(dir2 + "/" + lnames2[0], brlen, bclen);
                    boolean appendOnly = mb.isInSparseFormat();
                    DenseBlock compare = DataConverter.convertToDenseBlock(mb, false);
                    for (String lname : dir.list()) {
                        MatrixBlock tmp = StagingFileUtils.readCellList2BlockFromLocal(dir + "/" + lname, brlen, bclen);
                        mergeWithComp(mb, tmp, compare);
                    }
                    // sort sparse due to append-only
                    if (appendOnly && !_isAccum)
                        mb.sortSparseRows();
                    // change sparsity if required after
                    mb.examSparsity();
                } else // WITHOUT COMPARE BLOCK
                {
                    // copy all non-zeros from all workers
                    boolean appendOnly = false;
                    for (String lname : dir.list()) {
                        if (mb == null) {
                            mb = StagingFileUtils.readCellList2BlockFromLocal(dir + "/" + lname, brlen, bclen);
                            appendOnly = mb.isInSparseFormat();
                        } else {
                            MatrixBlock tmp = StagingFileUtils.readCellList2BlockFromLocal(dir + "/" + lname, brlen, bclen);
                            mergeWithoutComp(mb, tmp, appendOnly);
                        }
                    }
                    // sort sparse due to append-only
                    if (appendOnly && !_isAccum)
                        mb.sortSparseRows();
                    // change sparsity if required after
                    mb.examSparsity();
                }
            }
            // write the block to binary cell
            if (mb != null) {
                if (mb.isInSparseFormat()) {
                    Iterator<IJV> iter = mb.getSparseBlockIterator();
                    while (iter.hasNext()) {
                        IJV lcell = iter.next();
                        indexes.setIndexes(row_offset + lcell.getI(), col_offset + lcell.getJ());
                        cell.setValue(lcell.getV());
                        out.append(indexes, cell);
                        written = true;
                    }
                } else {
                    for (int i = 0; i < brlen; i++) for (int j = 0; j < bclen; j++) {
                        double lvalue = mb.getValueDenseUnsafe(i, j);
                        if (// for nnz
                        lvalue != 0) {
                            indexes.setIndexes(row_offset + i, col_offset + j);
                            cell.setValue(lvalue);
                            out.append(indexes, cell);
                            written = true;
                        }
                    }
                }
            }
        }
        if (!written)
            out.append(indexes, cell);
    } finally {
        IOUtilFunctions.closeSilently(out);
    }
}

Example 4 with DenseBlock

use of org.apache.sysml.runtime.matrix.data.DenseBlock in project incubator-systemml by apache.

the class ReaderTextCell method readRawTextCellMatrixFromInputStream.

private static void readRawTextCellMatrixFromInputStream(InputStream is, MatrixBlock dest, long rlen, long clen, int brlen, int bclen, boolean matrixMarket) throws IOException {
    BufferedReader br = new BufferedReader(new InputStreamReader(is));
    boolean sparse = dest.isInSparseFormat();
    String value = null;
    int row = -1;
    int col = -1;
    // Read the header lines, if reading from a matrixMarket file
    if (matrixMarket) {
        // header line
        value = br.readLine();
        if (value == null || !value.startsWith("%%")) {
            throw new IOException("Error while reading file in MatrixMarket format. Expecting a header line, but encountered, \"" + value + "\".");
        }
        // skip until end-of-comments
        while ((value = br.readLine()) != null && value.charAt(0) == '%') {
        // do nothing just skip comments
        }
        // the first line after comments is the one w/ matrix dimensions
        // validate (rlen clen nnz)
        String[] fields = value.trim().split("\\s+");
        long mm_rlen = Long.parseLong(fields[0]);
        long mm_clen = Long.parseLong(fields[1]);
        if (rlen != mm_rlen || clen != mm_clen) {
            throw new IOException("Unexpected matrix dimensions while reading file in MatrixMarket format. Expecting dimensions [" + rlen + " rows, " + clen + " cols] but encountered [" + mm_rlen + " rows, " + mm_clen + "cols].");
        }
    }
    try {
        FastStringTokenizer st = new FastStringTokenizer(' ');
        if (// SPARSE<-value
        sparse) {
            while ((value = br.readLine()) != null) {
                // reinit tokenizer
                st.reset(value);
                row = st.nextInt() - 1;
                col = st.nextInt() - 1;
                if (row == -1 || col == -1)
                    continue;
                double lvalue = st.nextDouble();
                dest.appendValue(row, col, lvalue);
            }
            dest.sortSparseRows();
        } else // DENSE<-value
        {
            DenseBlock a = dest.getDenseBlock();
            while ((value = br.readLine()) != null) {
                // reinit tokenizer
                st.reset(value);
                row = st.nextInt() - 1;
                col = st.nextInt() - 1;
                if (row == -1 || col == -1)
                    continue;
                double lvalue = st.nextDouble();
                a.set(row, col, lvalue);
            }
        }
    } catch (Exception ex) {
        // post-mortem error handling and bounds checking
        if (row < 0 || row + 1 > rlen || col < 0 || col + 1 > clen)
            throw new IOException("Matrix cell [" + (row + 1) + "," + (col + 1) + "] " + "out of overall matrix range [1:" + rlen + ",1:" + clen + "].", ex);
        else
            throw new IOException("Unable to read matrix in raw text cell format.", ex);
    } finally {
        IOUtilFunctions.closeSilently(br);
    }
}

Example 5 with DenseBlock

use of org.apache.sysml.runtime.matrix.data.DenseBlock in project incubator-systemml by apache.

the class SpoofOuterProduct method execute.

@Override
public MatrixBlock execute(ArrayList<MatrixBlock> inputs, ArrayList<ScalarObject> scalarObjects, MatrixBlock out, int numThreads) {
    // sanity check
    if (inputs == null || inputs.size() < 3 || out == null)
        throw new RuntimeException("Invalid input arguments.");
    // check empty result
    if (// U is empty
    (_outerProductType == OutProdType.LEFT_OUTER_PRODUCT && inputs.get(1).isEmptyBlock(false)) || // V is empty
    (_outerProductType == OutProdType.RIGHT_OUTER_PRODUCT && inputs.get(2).isEmptyBlock(false)) || inputs.get(0).isEmptyBlock(false)) {
        // X is empty
        // turn empty dense into sparse
        out.examSparsity();
        return out;
    }
    // input preparation and result allocation (Allocate the output that is set by Sigma2CPInstruction)
    if (_outerProductType == OutProdType.CELLWISE_OUTER_PRODUCT) {
        // assign it to the time and sparse representation of the major input matrix
        out.reset(inputs.get(0).getNumRows(), inputs.get(0).getNumColumns(), inputs.get(0).isInSparseFormat());
        out.allocateBlock();
    } else {
        // if left outerproduct gives a value of k*n instead of n*k, change it back to n*k and then transpose the output
        if (_outerProductType == OutProdType.LEFT_OUTER_PRODUCT)
            // n*k
            out.reset(inputs.get(0).getNumColumns(), inputs.get(1).getNumColumns(), false);
        else if (_outerProductType == OutProdType.RIGHT_OUTER_PRODUCT)
            // m*k
            out.reset(inputs.get(0).getNumRows(), inputs.get(1).getNumColumns(), false);
        out.allocateDenseBlock();
    }
    if (2 * inputs.get(0).getNonZeros() * inputs.get(1).getNumColumns() < PAR_MINFLOP_THRESHOLD)
        // sequential
        return execute(inputs, scalarObjects, out);
    // input preparation
    DenseBlock[] ab = getDenseMatrices(prepInputMatrices(inputs, 1, 2, true, false));
    SideInput[] b = prepInputMatrices(inputs, 3, false);
    double[] scalars = prepInputScalars(scalarObjects);
    // core sequential execute
    final int m = inputs.get(0).getNumRows();
    final int n = inputs.get(0).getNumColumns();
    // rank
    final int k = inputs.get(1).getNumColumns();
    final long nnz = inputs.get(0).getNonZeros();
    MatrixBlock a = inputs.get(0);
    try {
        ExecutorService pool = CommonThreadPool.get(numThreads);
        ArrayList<ParExecTask> tasks = new ArrayList<>();
        if (_outerProductType == OutProdType.LEFT_OUTER_PRODUCT) {
            if (a instanceof CompressedMatrixBlock) {
                // parallelize over column groups
                int numCG = ((CompressedMatrixBlock) a).getNumColGroups();
                int blklen = (int) (Math.ceil((double) numCG / numThreads));
                for (int j = 0; j < numThreads & j * blklen < numCG; j++) tasks.add(new ParExecTask(a, ab[0], ab[1], b, scalars, out, m, n, k, _outerProductType, 0, m, j * blklen, Math.min((j + 1) * blklen, numCG)));
            } else {
                // parallelize over column partitions
                int blklen = (int) (Math.ceil((double) n / numThreads));
                for (int j = 0; j < numThreads & j * blklen < n; j++) tasks.add(new ParExecTask(a, ab[0], ab[1], b, scalars, out, m, n, k, _outerProductType, 0, m, j * blklen, Math.min((j + 1) * blklen, n)));
            }
        } else {
            // right or cell-wise
            // parallelize over row partitions
            int numThreads2 = getPreferredNumberOfTasks(m, n, nnz, k, numThreads);
            int blklen = (int) (Math.ceil((double) m / numThreads2));
            for (int i = 0; i < numThreads2 & i * blklen < m; i++) tasks.add(new ParExecTask(a, ab[0], ab[1], b, scalars, out, m, n, k, _outerProductType, i * blklen, Math.min((i + 1) * blklen, m), 0, n));
        }
        List<Future<Long>> taskret = pool.invokeAll(tasks);
        pool.shutdown();
        for (Future<Long> task : taskret) out.setNonZeros(out.getNonZeros() + task.get());
    } catch (Exception e) {
        throw new DMLRuntimeException(e);
    }
    // post-processing
    if (a instanceof CompressedMatrixBlock) {
        if (out.isInSparseFormat() && _outerProductType == OutProdType.CELLWISE_OUTER_PRODUCT)
            out.sortSparseRows();
        else if (_outerProductType == OutProdType.LEFT_OUTER_PRODUCT)
            out.recomputeNonZeros();
    }
    out.examSparsity();
    return out;
}