Examples with IJV org.apache.sysml.runtime.matrix.data.IJV used on opensource projects

Example 6 with IJV

use of org.apache.sysml.runtime.matrix.data.IJV in project incubator-systemml by apache.

the class ResultMergeLocalFile method createBinaryCellResultFile.

@SuppressWarnings("deprecation")
private void createBinaryCellResultFile(String fnameStaging, String fnameStagingCompare, String fnameNew, MetaDataFormat metadata, boolean withCompare) throws IOException, DMLRuntimeException {
    JobConf job = new JobConf(ConfigurationManager.getCachedJobConf());
    Path path = new Path(fnameNew);
    FileSystem fs = IOUtilFunctions.getFileSystem(path, job);
    MatrixCharacteristics mc = metadata.getMatrixCharacteristics();
    long rlen = mc.getRows();
    long clen = mc.getCols();
    int brlen = mc.getRowsPerBlock();
    int bclen = mc.getColsPerBlock();
    MatrixIndexes indexes = new MatrixIndexes(1, 1);
    MatrixCell cell = new MatrixCell(0);
    // beware ca 50ms
    SequenceFile.Writer out = new SequenceFile.Writer(fs, job, path, MatrixIndexes.class, MatrixCell.class);
    try {
        boolean written = false;
        for (long brow = 1; brow <= (long) Math.ceil(rlen / (double) brlen); brow++) for (long bcol = 1; bcol <= (long) Math.ceil(clen / (double) bclen); bcol++) {
            File dir = new File(fnameStaging + "/" + brow + "_" + bcol);
            File dir2 = new File(fnameStagingCompare + "/" + brow + "_" + bcol);
            MatrixBlock mb = null;
            long row_offset = (brow - 1) * brlen + 1;
            long col_offset = (bcol - 1) * bclen + 1;
            if (dir.exists()) {
                if (// WITH COMPARE BLOCK
                withCompare && dir2.exists()) {
                    // copy only values that are different from the original
                    String[] lnames2 = dir2.list();
                    if (// there should be exactly 1 compare block
                    lnames2.length != 1)
                        throw new DMLRuntimeException("Unable to merge results because multiple compare blocks found.");
                    mb = StagingFileUtils.readCellList2BlockFromLocal(dir2 + "/" + lnames2[0], brlen, bclen);
                    boolean appendOnly = mb.isInSparseFormat();
                    DenseBlock compare = DataConverter.convertToDenseBlock(mb, false);
                    for (String lname : dir.list()) {
                        MatrixBlock tmp = StagingFileUtils.readCellList2BlockFromLocal(dir + "/" + lname, brlen, bclen);
                        mergeWithComp(mb, tmp, compare);
                    }
                    // sort sparse due to append-only
                    if (appendOnly && !_isAccum)
                        mb.sortSparseRows();
                    // change sparsity if required after
                    mb.examSparsity();
                } else // WITHOUT COMPARE BLOCK
                {
                    // copy all non-zeros from all workers
                    boolean appendOnly = false;
                    for (String lname : dir.list()) {
                        if (mb == null) {
                            mb = StagingFileUtils.readCellList2BlockFromLocal(dir + "/" + lname, brlen, bclen);
                            appendOnly = mb.isInSparseFormat();
                        } else {
                            MatrixBlock tmp = StagingFileUtils.readCellList2BlockFromLocal(dir + "/" + lname, brlen, bclen);
                            mergeWithoutComp(mb, tmp, appendOnly);
                        }
                    }
                    // sort sparse due to append-only
                    if (appendOnly && !_isAccum)
                        mb.sortSparseRows();
                    // change sparsity if required after
                    mb.examSparsity();
                }
            }
            // write the block to binary cell
            if (mb != null) {
                if (mb.isInSparseFormat()) {
                    Iterator<IJV> iter = mb.getSparseBlockIterator();
                    while (iter.hasNext()) {
                        IJV lcell = iter.next();
                        indexes.setIndexes(row_offset + lcell.getI(), col_offset + lcell.getJ());
                        cell.setValue(lcell.getV());
                        out.append(indexes, cell);
                        written = true;
                    }
                } else {
                    for (int i = 0; i < brlen; i++) for (int j = 0; j < bclen; j++) {
                        double lvalue = mb.getValueDenseUnsafe(i, j);
                        if (// for nnz
                        lvalue != 0) {
                            indexes.setIndexes(row_offset + i, col_offset + j);
                            cell.setValue(lvalue);
                            out.append(indexes, cell);
                            written = true;
                        }
                    }
                }
            }
        }
        if (!written)
            out.append(indexes, cell);
    } finally {
        IOUtilFunctions.closeSilently(out);
    }
}

Example 7 with IJV

use of org.apache.sysml.runtime.matrix.data.IJV in project incubator-systemml by apache.

the class SpoofCellwise method executeCompressedColAggSum.

private long executeCompressedColAggSum(CompressedMatrixBlock a, SideInput[] b, double[] scalars, double[] c, int m, int n, boolean sparseSafe, int rl, int ru) {
    KahanFunction kplus = (KahanFunction) getAggFunction();
    KahanObject kbuff = new KahanObject(0, 0);
    double[] corr = new double[n];
    Iterator<IJV> iter = a.getIterator(rl, ru, !sparseSafe);
    while (iter.hasNext()) {
        IJV cell = iter.next();
        double val = genexec(cell.getV(), b, scalars, m, n, cell.getI(), cell.getJ());
        kbuff.set(c[cell.getJ()], corr[cell.getJ()]);
        kplus.execute2(kbuff, val);
        c[cell.getJ()] = kbuff._sum;
        corr[cell.getJ()] = kbuff._correction;
    }
    return -1;
}

Example 8 with IJV

use of org.apache.sysml.runtime.matrix.data.IJV in project incubator-systemml by apache.

the class SpoofCellwise method executeCompressedAggSum.

private double executeCompressedAggSum(CompressedMatrixBlock a, SideInput[] b, double[] scalars, int m, int n, boolean sparseSafe, int rl, int ru) {
    KahanFunction kplus = (KahanFunction) getAggFunction();
    KahanObject kbuff = new KahanObject(0, 0);
    KahanObject kbuff2 = new KahanObject(0, 0);
    // special case: computation over value-tuples only
    if (sparseSafe && b.length == 0 && !a.hasUncompressedColGroup()) {
        // note: all remaining groups are guaranteed ColGroupValue
        boolean entireGrp = (rl == 0 && ru == a.getNumRows());
        int maxNumVals = a.getColGroups().stream().mapToInt(g -> ((ColGroupValue) g).getNumValues()).max().orElse(0);
        int[] counts = new int[maxNumVals];
        for (ColGroup grp : a.getColGroups()) {
            ColGroupValue grpv = (ColGroupValue) grp;
            counts = entireGrp ? grpv.getCounts(counts) : grpv.getCounts(rl, ru, counts);
            for (int k = 0; k < grpv.getNumValues(); k++) {
                kbuff2.set(0, 0);
                double in = grpv.sumValues(k, kplus, kbuff2);
                double out = genexec(in, b, scalars, m, n, -1, -1);
                kplus.execute3(kbuff, out, counts[k]);
            }
        }
    } else // general case of arbitrary side inputs
    {
        Iterator<IJV> iter = a.getIterator(rl, ru, !sparseSafe);
        while (iter.hasNext()) {
            IJV cell = iter.next();
            double val = genexec(cell.getV(), b, scalars, m, n, cell.getI(), cell.getJ());
            kplus.execute2(kbuff, val);
        }
    }
    return kbuff._sum;
}

Example 9 with IJV

use of org.apache.sysml.runtime.matrix.data.IJV in project incubator-systemml by apache.

the class CumSumProd method addC.

private void addC(int i, boolean addPrevRow) {
    boolean isCVector = C.getNumColumns() != ret.getNumCols();
    if (C.isInSparseFormat()) {
        Iterator<IJV> iter = C.getSparseBlockIterator(i, i + 1);
        while (iter.hasNext()) {
            IJV ijv = iter.next();
            if (!isCVector) {
                if (addPrevRow)
                    denseBlock[ijv.getI() * numRetCols + ijv.getJ()] += ijv.getV() * denseBlock[(ijv.getI() - 1) * numRetCols + ijv.getJ()];
                else
                    denseBlock[ijv.getI() * numRetCols + ijv.getJ()] += ijv.getV() * denseBlock[(ijv.getI() + 1) * numRetCols + ijv.getJ()];
            } else {
                double val = ijv.getV();
                for (int j = ijv.getI() * numRetCols; j < (ijv.getI() + 1) * numRetCols; j++) {
                    double val1 = addPrevRow ? denseBlock[(ijv.getI() - 1) * numRetCols + ijv.getJ()] : denseBlock[(ijv.getI() + 1) * numRetCols + ijv.getJ()];
                    denseBlock[j] += val * val1;
                }
            }
        }
    } else {
        double[] CBlk = C.getDenseBlockValues();
        if (CBlk != null) {
            if (!isCVector) {
                for (int j = i * numRetCols; j < (i + 1) * numRetCols; j++) {
                    double val1 = addPrevRow ? denseBlock[j - numRetCols] : denseBlock[j + numRetCols];
                    denseBlock[j] += CBlk[j] * val1;
                }
            } else {
                for (int j = i * numRetCols; j < (i + 1) * numRetCols; j++) {
                    double val1 = addPrevRow ? denseBlock[j - numRetCols] : denseBlock[j + numRetCols];
                    denseBlock[j] += CBlk[i] * val1;
                }
            }
        }
    }
}

Example 10 with IJV

use of org.apache.sysml.runtime.matrix.data.IJV in project incubator-systemml by apache.

the class CumSumProd method execute.

@Override
public void execute() {
    X = ((Matrix) getFunctionInput(0)).getMatrixObject().acquireRead();
    C = ((Matrix) getFunctionInput(1)).getMatrixObject().acquireRead();
    if (X.getNumRows() != C.getNumRows())
        throw new RuntimeException("Number of rows of X and C should match");
    if (X.getNumColumns() != C.getNumColumns() && C.getNumColumns() != 1)
        throw new RuntimeException("Incorrect Number of columns of X and C (Expected C to be of same dimension or a vector)");
    start = Double.parseDouble(((Scalar) getFunctionInput(2)).getValue());
    isReverse = Boolean.parseBoolean(((Scalar) getFunctionInput(3)).getValue());
    numRetRows = X.getNumRows();
    numRetCols = X.getNumColumns();
    allocateOutput();
    // Copy X to Y
    denseBlock = retMB.getDenseBlockValues();
    if (X.isInSparseFormat()) {
        Iterator<IJV> iter = X.getSparseBlockIterator();
        while (iter.hasNext()) {
            IJV ijv = iter.next();
            denseBlock[ijv.getI() * numRetCols + ijv.getJ()] = ijv.getV();
        }
    } else {
        if (X.getDenseBlock() != null)
            System.arraycopy(X.getDenseBlockValues(), 0, denseBlock, 0, denseBlock.length);
    }
    if (!isReverse) {
        // Y [1, ] = X [1, ] + C [1, ] * start;
        // Y [i+1, ] = X [i+1, ] + C [i+1, ] * Y [i, ]
        addCNConstant(0, start);
        for (int i = 1; i < numRetRows; i++) {
            addC(i, true);
        }
    } else {
        // Y [m, ] = X [m, ] + C [m, ] * start;
        // Y [i-1, ] = X [i-1, ] + C [i-1, ] * Y [i, ]
        addCNConstant(numRetRows - 1, start);
        for (int i = numRetRows - 2; i >= 0; i--) {
            addC(i, false);
        }
    }
    ((Matrix) getFunctionInput(1)).getMatrixObject().release();
    ((Matrix) getFunctionInput(0)).getMatrixObject().release();
    retMB.recomputeNonZeros();
    try {
        retMB.examSparsity();
        ret.setMatrixDoubleArray(retMB, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo);
    } catch (DMLRuntimeException e) {
        throw new RuntimeException("Error while executing CumSumProd", e);
    } catch (IOException e) {
        throw new RuntimeException("Error while executing CumSumProd", e);
    }
}