Examples with SparseBlock org.apache.sysml.runtime.matrix.data.SparseBlock used on opensource projects

Example 1 with SparseBlock

use of org.apache.sysml.runtime.matrix.data.SparseBlock in project incubator-systemml by apache.

the class SpoofCellwise method executeSparseNoAggSparse.

private long executeSparseNoAggSparse(SparseBlock sblock, SideInput[] b, double[] scalars, MatrixBlock out, int m, int n, boolean sparseSafe, int rl, int ru) throws DMLRuntimeException {
    //note: sequential scan algorithm for both sparse-safe and -unsafe 
    //in order to avoid binary search for sparse-unsafe 
    SparseBlock c = out.getSparseBlock();
    long lnnz = 0;
    for (int i = rl; i < ru; i++) {
        int lastj = -1;
        //handle non-empty rows
        if (sblock != null && !sblock.isEmpty(i)) {
            int apos = sblock.pos(i);
            int alen = sblock.size(i);
            int[] aix = sblock.indexes(i);
            double[] avals = sblock.values(i);
            c.allocate(i, sparseSafe ? alen : n);
            for (int k = apos; k < apos + alen; k++) {
                //process zeros before current non-zero
                if (!sparseSafe)
                    for (int j = lastj + 1; j < aix[k]; j++) c.append(i, j, genexec(0, b, scalars, m, n, i, j));
                //process current non-zero
                lastj = aix[k];
                c.append(i, lastj, genexec(avals[k], b, scalars, m, n, i, lastj));
            }
        }
        //process empty rows or remaining zeros
        if (!sparseSafe)
            for (int j = lastj + 1; j < n; j++) c.append(i, j, genexec(0, b, scalars, m, n, i, j));
        lnnz += c.size(i);
    }
    return lnnz;
}

Example 2 with SparseBlock

use of org.apache.sysml.runtime.matrix.data.SparseBlock in project incubator-systemml by apache.

the class SpoofOuterProduct method executeCellwiseSparse.

private void executeCellwiseSparse(SparseBlock sblock, double[] u, double[] v, double[][] b, double[] scalars, MatrixBlock out, int m, int n, int k, long nnz, OutProdType type, int rl, int ru, int cl, int cu) {
    final int blocksizeIJ = (int) (8L * m * n / nnz);
    int[] curk = new int[blocksizeIJ];
    if (//DENSE
    !out.isInSparseFormat()) {
        double[] c = out.getDenseBlock();
        for (int bi = rl; bi < ru; bi += blocksizeIJ) {
            int bimin = Math.min(ru, bi + blocksizeIJ);
            //prepare starting indexes for block row
            Arrays.fill(curk, 0);
            //blocked execution over column blocks
            for (int bj = 0; bj < n; bj += blocksizeIJ) {
                int bjmin = Math.min(n, bj + blocksizeIJ);
                for (int i = bi, uix = bi * k; i < bimin; i++, uix += k) {
                    if (sblock.isEmpty(i))
                        continue;
                    int wpos = sblock.pos(i);
                    int wlen = sblock.size(i);
                    int[] wix = sblock.indexes(i);
                    double[] wval = sblock.values(i);
                    int index = wpos + curk[i - bi];
                    for (; index < wpos + wlen && wix[index] < bjmin; index++) {
                        if (type == OutProdType.CELLWISE_OUTER_PRODUCT)
                            c[wix[index]] = genexecCellwise(wval[index], u, uix, v, wix[index] * k, b, scalars, m, n, k, i, wix[index]);
                        else
                            c[0] += genexecCellwise(wval[index], u, uix, v, wix[index] * k, b, scalars, m, n, k, i, wix[index]);
                    }
                    curk[i - bi] = index - wpos;
                }
            }
        }
    } else //SPARSE
    {
        SparseBlock c = out.getSparseBlock();
        for (int bi = rl; bi < ru; bi += blocksizeIJ) {
            int bimin = Math.min(ru, bi + blocksizeIJ);
            //prepare starting indexes for block row
            Arrays.fill(curk, 0);
            //blocked execution over column blocks
            for (int bj = 0; bj < n; bj += blocksizeIJ) {
                int bjmin = Math.min(n, bj + blocksizeIJ);
                for (int i = bi, uix = bi * k; i < bimin; i++, uix += k) {
                    if (sblock.isEmpty(i))
                        continue;
                    int wpos = sblock.pos(i);
                    int wlen = sblock.size(i);
                    int[] wix = sblock.indexes(i);
                    double[] wval = sblock.values(i);
                    int index = wpos + curk[i - bi];
                    for (; index < wpos + wlen && wix[index] < bjmin; index++) {
                        c.append(i, wix[index], genexecCellwise(wval[index], u, uix, v, wix[index] * k, b, scalars, m, n, k, i, wix[index]));
                    }
                    curk[i - bi] = index - wpos;
                }
            }
        }
    }
}

Example 3 with SparseBlock

use of org.apache.sysml.runtime.matrix.data.SparseBlock in project incubator-systemml by apache.

the class SpoofOuterProduct method executeCellwiseCompressed.

private void executeCellwiseCompressed(CompressedMatrixBlock a, double[] u, double[] v, double[][] b, double[] scalars, MatrixBlock out, int m, int n, int k, OutProdType type, int rl, int ru, int cl, int cu) {
    double[] c = out.getDenseBlock();
    SparseBlock csblock = out.getSparseBlock();
    Iterator<IJV> iter = a.getIterator(rl, ru, false);
    while (iter.hasNext()) {
        IJV cell = iter.next();
        int uix = cell.getI() * k;
        int vix = cell.getJ() * k;
        if (type == OutProdType.CELLWISE_OUTER_PRODUCT) {
            if (out.isInSparseFormat()) {
                csblock.allocate(cell.getI());
                csblock.append(cell.getI(), cell.getJ(), genexecCellwise(cell.getV(), u, uix, v, vix, b, scalars, m, n, k, cell.getI(), cell.getJ()));
            } else {
                c[cell.getI() * n + cell.getJ()] = genexecCellwise(cell.getV(), u, uix, v, vix, b, scalars, m, n, k, cell.getI(), cell.getJ());
            }
        } else {
            c[0] += genexecCellwise(cell.getV(), u, uix, v, vix, b, scalars, m, n, k, cell.getI(), cell.getJ());
        }
    }
}

Example 4 with SparseBlock

use of org.apache.sysml.runtime.matrix.data.SparseBlock in project incubator-systemml by apache.

the class BitmapEncoder method extractBitmap.

private static UncompressedBitmap extractBitmap(int colIndex, MatrixBlock rawblock, boolean skipZeros) {
    //probe map for distinct items (for value or value groups)
    DoubleIntListHashMap distinctVals = new DoubleIntListHashMap();
    //scan rows and probe/build distinct items
    final int m = CompressedMatrixBlock.TRANSPOSE_INPUT ? rawblock.getNumColumns() : rawblock.getNumRows();
    if (//SPARSE 
    rawblock.isInSparseFormat() && CompressedMatrixBlock.TRANSPOSE_INPUT) {
        SparseBlock a = rawblock.getSparseBlock();
        if (a != null && !a.isEmpty(colIndex)) {
            int apos = a.pos(colIndex);
            int alen = a.size(colIndex);
            int[] aix = a.indexes(colIndex);
            double[] avals = a.values(colIndex);
            //for 0 values
            IntArrayList lstPtr0 = new IntArrayList();
            int last = -1;
            //iterate over non-zero entries but fill in zeros
            for (int j = apos; j < apos + alen; j++) {
                //fill in zero values
                if (!skipZeros)
                    for (int k = last + 1; k < aix[j]; k++) lstPtr0.appendValue(k);
                //handle non-zero value
                IntArrayList lstPtr = distinctVals.get(avals[j]);
                if (lstPtr == null) {
                    lstPtr = new IntArrayList();
                    distinctVals.appendValue(avals[j], lstPtr);
                }
                lstPtr.appendValue(aix[j]);
                last = aix[j];
            }
            //fill in remaining zero values
            if (!skipZeros) {
                for (int k = last + 1; k < m; k++) lstPtr0.appendValue(k);
                if (lstPtr0.size() > 0)
                    distinctVals.appendValue(0, lstPtr0);
            }
        } else if (!skipZeros) {
            //full 0 column 
            IntArrayList lstPtr = new IntArrayList();
            for (int i = 0; i < m; i++) lstPtr.appendValue(i);
            distinctVals.appendValue(0, lstPtr);
        }
    } else //GENERAL CASE
    {
        for (int i = 0; i < m; i++) {
            double val = CompressedMatrixBlock.TRANSPOSE_INPUT ? rawblock.quickGetValue(colIndex, i) : rawblock.quickGetValue(i, colIndex);
            if (val != 0 || !skipZeros) {
                IntArrayList lstPtr = distinctVals.get(val);
                if (lstPtr == null) {
                    lstPtr = new IntArrayList();
                    distinctVals.appendValue(val, lstPtr);
                }
                lstPtr.appendValue(i);
            }
        }
    }
    return new UncompressedBitmap(distinctVals);
}

Example 5 with SparseBlock

use of org.apache.sysml.runtime.matrix.data.SparseBlock in project incubator-systemml by apache.

the class WriterTextCSV method writeCSVMatrixToFile.

protected final void writeCSVMatrixToFile(Path path, JobConf job, FileSystem fs, MatrixBlock src, int rl, int ru, CSVFileFormatProperties props) throws IOException {
    boolean sparse = src.isInSparseFormat();
    int clen = src.getNumColumns();
    //create buffered writer
    BufferedWriter br = new BufferedWriter(new OutputStreamWriter(fs.create(path, true)));
    try {
        //for obj reuse and preventing repeated buffer re-allocations
        StringBuilder sb = new StringBuilder();
        props = (props == null) ? new CSVFileFormatProperties() : props;
        String delim = props.getDelim();
        boolean csvsparse = props.isSparse();
        // Write header line, if needed
        if (props.hasHeader() && rl == 0) {
            //write row chunk-wise to prevent OOM on large number of columns
            for (int bj = 0; bj < clen; bj += BLOCKSIZE_J) {
                for (int j = bj; j < Math.min(clen, bj + BLOCKSIZE_J); j++) {
                    sb.append("C" + (j + 1));
                    if (j < clen - 1)
                        sb.append(delim);
                }
                br.write(sb.toString());
                sb.setLength(0);
            }
            sb.append('\n');
            br.write(sb.toString());
            sb.setLength(0);
        }
        // Write data lines
        if (//SPARSE
        sparse) {
            SparseBlock sblock = src.getSparseBlock();
            for (int i = rl; i < ru; i++) {
                //write row chunk-wise to prevent OOM on large number of columns
                int prev_jix = -1;
                if (sblock != null && i < sblock.numRows() && !sblock.isEmpty(i)) {
                    int pos = sblock.pos(i);
                    int alen = sblock.size(i);
                    int[] aix = sblock.indexes(i);
                    double[] avals = sblock.values(i);
                    for (int j = pos; j < pos + alen; j++) {
                        int jix = aix[j];
                        // output empty fields, if needed
                        for (int j2 = prev_jix; j2 < jix - 1; j2++) {
                            if (!csvsparse)
                                sb.append('0');
                            sb.append(delim);
                            //flush buffered string
                            if (j2 % BLOCKSIZE_J == 0) {
                                br.write(sb.toString());
                                sb.setLength(0);
                            }
                        }
                        // output the value (non-zero)
                        sb.append(avals[j]);
                        if (jix < clen - 1)
                            sb.append(delim);
                        br.write(sb.toString());
                        sb.setLength(0);
                        //flush buffered string
                        if (jix % BLOCKSIZE_J == 0) {
                            br.write(sb.toString());
                            sb.setLength(0);
                        }
                        prev_jix = jix;
                    }
                }
                // In case of an empty row, output (clen-1) empty fields
                for (int bj = prev_jix + 1; bj < clen; bj += BLOCKSIZE_J) {
                    for (int j = bj; j < Math.min(clen, bj + BLOCKSIZE_J); j++) {
                        if (!csvsparse)
                            sb.append('0');
                        if (j < clen - 1)
                            sb.append(delim);
                    }
                    br.write(sb.toString());
                    sb.setLength(0);
                }
                sb.append('\n');
                br.write(sb.toString());
                sb.setLength(0);
            }
        } else //DENSE
        {
            for (int i = rl; i < ru; i++) {
                //write row chunk-wise to prevent OOM on large number of columns
                for (int bj = 0; bj < clen; bj += BLOCKSIZE_J) {
                    for (int j = bj; j < Math.min(clen, bj + BLOCKSIZE_J); j++) {
                        double lvalue = src.getValueDenseUnsafe(i, j);
                        if (//for nnz
                        lvalue != 0)
                            sb.append(lvalue);
                        else if (!csvsparse)
                            sb.append('0');
                        if (j != clen - 1)
                            sb.append(delim);
                    }
                    br.write(sb.toString());
                    sb.setLength(0);
                }
                sb.append('\n');
                //same as append
                br.write(sb.toString());
                sb.setLength(0);
            }
        }
    } finally {
        IOUtilFunctions.closeSilently(br);
    }
}