Examples with CompressedMatrixBlock org.apache.sysml.runtime.compress.CompressedMatrixBlock used on opensource projects

Example 31 with CompressedMatrixBlock

use of org.apache.sysml.runtime.compress.CompressedMatrixBlock in project incubator-systemml by apache.

the class ParMatrixMultChainTest method runMatrixMultChainTest.

private static void runMatrixMultChainTest(SparsityType sptype, ValueType vtype, ChainType ctype, boolean compress) {
    try {
        // prepare sparsity for input data
        double sparsity = -1;
        switch(sptype) {
            case DENSE:
                sparsity = sparsity1;
                break;
            case SPARSE:
                sparsity = sparsity2;
                break;
            case EMPTY:
                sparsity = sparsity3;
                break;
        }
        // generate input data
        double min = (vtype == ValueType.CONST) ? 10 : -10;
        double[][] input = TestUtils.generateTestMatrix(rows, cols, min, 10, sparsity, 7);
        if (vtype == ValueType.RAND_ROUND_OLE || vtype == ValueType.RAND_ROUND_DDC) {
            CompressedMatrixBlock.ALLOW_DDC_ENCODING = (vtype == ValueType.RAND_ROUND_DDC);
            input = TestUtils.round(input);
        }
        MatrixBlock mb = DataConverter.convertToMatrixBlock(input);
        MatrixBlock vector1 = DataConverter.convertToMatrixBlock(TestUtils.generateTestMatrix(cols, 1, 0, 1, 1.0, 3));
        MatrixBlock vector2 = (ctype == ChainType.XtwXv) ? DataConverter.convertToMatrixBlock(TestUtils.generateTestMatrix(rows, 1, 0, 1, 1.0, 3)) : null;
        // compress given matrix block
        CompressedMatrixBlock cmb = new CompressedMatrixBlock(mb);
        if (compress)
            cmb.compress();
        // matrix-vector uncompressed
        int k = InfrastructureAnalyzer.getLocalParallelism();
        MatrixBlock ret1 = (MatrixBlock) mb.chainMatrixMultOperations(vector1, vector2, new MatrixBlock(), ctype, k);
        // matrix-vector compressed
        MatrixBlock ret2 = (MatrixBlock) cmb.chainMatrixMultOperations(vector1, vector2, new MatrixBlock(), ctype, k);
        // compare result with input
        double[][] d1 = DataConverter.convertToDoubleMatrix(ret1);
        double[][] d2 = DataConverter.convertToDoubleMatrix(ret2);
        TestUtils.compareMatrices(d1, d2, cols, 1, 0.0000001);
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    } finally {
        CompressedMatrixBlock.ALLOW_DDC_ENCODING = true;
    }
}

Example 32 with CompressedMatrixBlock

use of org.apache.sysml.runtime.compress.CompressedMatrixBlock in project incubator-systemml by apache.

the class ParTransposeSelfLeftMatrixMultTest method runTransposeSelfMatrixMultTest.

private static void runTransposeSelfMatrixMultTest(SparsityType sptype, ValueType vtype, boolean compress) {
    try {
        // prepare sparsity for input data
        double sparsity = -1;
        switch(sptype) {
            case DENSE:
                sparsity = sparsity1;
                break;
            case SPARSE:
                sparsity = sparsity2;
                break;
            case EMPTY:
                sparsity = sparsity3;
                break;
        }
        // generate input data
        double min = (vtype == ValueType.CONST) ? 10 : -10;
        double[][] input = TestUtils.generateTestMatrix(rows, cols, min, 10, sparsity, 7);
        if (vtype == ValueType.RAND_ROUND_OLE || vtype == ValueType.RAND_ROUND_DDC) {
            CompressedMatrixBlock.ALLOW_DDC_ENCODING = (vtype == ValueType.RAND_ROUND_DDC);
            input = TestUtils.round(input);
        }
        MatrixBlock mb = DataConverter.convertToMatrixBlock(input);
        // compress given matrix block
        CompressedMatrixBlock cmb = new CompressedMatrixBlock(mb);
        if (compress)
            cmb.compress();
        // matrix-vector uncompressed
        int k = InfrastructureAnalyzer.getLocalParallelism();
        MatrixBlock ret1 = mb.transposeSelfMatrixMultOperations(new MatrixBlock(), MMTSJType.LEFT, k);
        // matrix-vector compressed
        MatrixBlock ret2 = cmb.transposeSelfMatrixMultOperations(new MatrixBlock(), MMTSJType.LEFT, k);
        // compare result with input
        double[][] d1 = DataConverter.convertToDoubleMatrix(ret1);
        double[][] d2 = DataConverter.convertToDoubleMatrix(ret2);
        TestUtils.compareMatrices(d1, d2, cols, cols, 0.0000001);
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    } finally {
        CompressedMatrixBlock.ALLOW_DDC_ENCODING = true;
    }
}

Example 33 with CompressedMatrixBlock

use of org.apache.sysml.runtime.compress.CompressedMatrixBlock in project incubator-systemml by apache.

the class ParUnaryAggregateTest method runUnaryAggregateTest.

private static void runUnaryAggregateTest(SparsityType sptype, ValueType vtype, AggType aggtype, boolean compress) {
    try {
        // prepare sparsity for input data
        double sparsity = -1;
        switch(sptype) {
            case DENSE:
                sparsity = sparsity1;
                break;
            case SPARSE:
                sparsity = sparsity2;
                break;
            case EMPTY:
                sparsity = sparsity3;
                break;
        }
        // generate input data
        double min = (vtype == ValueType.CONST) ? 10 : -10;
        double[][] input = TestUtils.generateTestMatrix(rows, cols1, min, 10, sparsity, 7);
        if (vtype == ValueType.RAND_ROUND_OLE || vtype == ValueType.RAND_ROUND_DDC) {
            CompressedMatrixBlock.ALLOW_DDC_ENCODING = (vtype == ValueType.RAND_ROUND_DDC);
            input = TestUtils.round(input);
        }
        MatrixBlock mb = DataConverter.convertToMatrixBlock(input);
        // uc group
        mb = mb.append(MatrixBlock.seqOperations(0.1, rows - 0.1, 1), new MatrixBlock());
        // prepare unary aggregate operator
        AggregateUnaryOperator auop = null;
        int k = InfrastructureAnalyzer.getLocalParallelism();
        switch(aggtype) {
            case SUM:
                auop = InstructionUtils.parseBasicAggregateUnaryOperator("uak+", k);
                break;
            case ROWSUMS:
                auop = InstructionUtils.parseBasicAggregateUnaryOperator("uark+", k);
                break;
            case COLSUMS:
                auop = InstructionUtils.parseBasicAggregateUnaryOperator("uack+", k);
                break;
            case SUMSQ:
                auop = InstructionUtils.parseBasicAggregateUnaryOperator("uasqk+", k);
                break;
            case ROWSUMSSQ:
                auop = InstructionUtils.parseBasicAggregateUnaryOperator("uarsqk+", k);
                break;
            case COLSUMSSQ:
                auop = InstructionUtils.parseBasicAggregateUnaryOperator("uacsqk+", k);
                break;
            case MAX:
                auop = InstructionUtils.parseBasicAggregateUnaryOperator("uamax", k);
                break;
            case ROWMAXS:
                auop = InstructionUtils.parseBasicAggregateUnaryOperator("uarmax", k);
                break;
            case COLMAXS:
                auop = InstructionUtils.parseBasicAggregateUnaryOperator("uacmax", k);
                break;
            case MIN:
                auop = InstructionUtils.parseBasicAggregateUnaryOperator("uamin", k);
                break;
            case ROWMINS:
                auop = InstructionUtils.parseBasicAggregateUnaryOperator("uarmin", k);
                break;
            case COLMINS:
                auop = InstructionUtils.parseBasicAggregateUnaryOperator("uacmin", k);
                break;
        }
        // compress given matrix block
        CompressedMatrixBlock cmb = new CompressedMatrixBlock(mb);
        if (compress)
            cmb.compress();
        // matrix-vector uncompressed
        MatrixBlock ret1 = (MatrixBlock) mb.aggregateUnaryOperations(auop, new MatrixBlock(), 1000, 1000, null, true);
        // matrix-vector compressed
        MatrixBlock ret2 = (MatrixBlock) cmb.aggregateUnaryOperations(auop, new MatrixBlock(), 1000, 1000, null, true);
        // compare result with input
        double[][] d1 = DataConverter.convertToDoubleMatrix(ret1);
        double[][] d2 = DataConverter.convertToDoubleMatrix(ret2);
        int dim1 = (aggtype == AggType.ROWSUMS || aggtype == AggType.ROWSUMSSQ || aggtype == AggType.ROWMINS || aggtype == AggType.ROWMINS) ? rows : 1;
        int dim2 = (aggtype == AggType.COLSUMS || aggtype == AggType.COLSUMSSQ || aggtype == AggType.COLMAXS || aggtype == AggType.COLMINS) ? cols1 : 1;
        TestUtils.compareMatrices(d1, d2, dim1, dim2, 0.00000000001);
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    } finally {
        CompressedMatrixBlock.ALLOW_DDC_ENCODING = true;
    }
}

Example 34 with CompressedMatrixBlock

use of org.apache.sysml.runtime.compress.CompressedMatrixBlock in project incubator-systemml by apache.

the class SpoofCellwise method execute.

@Override
public ScalarObject execute(ArrayList<MatrixBlock> inputs, ArrayList<ScalarObject> scalarObjects, int k) {
    // sanity check
    if (inputs == null || inputs.size() < 1)
        throw new RuntimeException("Invalid input arguments.");
    // input preparation
    MatrixBlock a = inputs.get(0);
    SideInput[] b = prepInputMatrices(inputs);
    double[] scalars = prepInputScalars(scalarObjects);
    final int m = a.getNumRows();
    final int n = a.getNumColumns();
    // sparse safe check
    boolean sparseSafe = isSparseSafe() || (b.length == 0 && genexec(0, b, scalars, m, n, 0, 0) == 0);
    long inputSize = sparseSafe ? getTotalInputNnz(inputs) : getTotalInputSize(inputs);
    if (inputSize < PAR_NUMCELL_THRESHOLD) {
        // serial execution
        k = 1;
    }
    double ret = 0;
    if (// SINGLE-THREADED
    k <= 1) {
        if (inputs.get(0) instanceof CompressedMatrixBlock)
            ret = executeCompressedAndAgg((CompressedMatrixBlock) a, b, scalars, m, n, sparseSafe, 0, m);
        else if (!inputs.get(0).isInSparseFormat())
            ret = executeDenseAndAgg(a.getDenseBlock(), b, scalars, m, n, sparseSafe, 0, m);
        else
            ret = executeSparseAndAgg(a.getSparseBlock(), b, scalars, m, n, sparseSafe, 0, m);
    } else // MULTI-THREADED
    {
        try {
            ExecutorService pool = CommonThreadPool.get(k);
            ArrayList<ParAggTask> tasks = new ArrayList<>();
            int nk = (a instanceof CompressedMatrixBlock) ? k : UtilFunctions.roundToNext(Math.min(8 * k, m / 32), k);
            int blklen = (int) (Math.ceil((double) m / nk));
            if (a instanceof CompressedMatrixBlock)
                blklen = BitmapEncoder.getAlignedBlocksize(blklen);
            for (int i = 0; i < nk & i * blklen < m; i++) tasks.add(new ParAggTask(a, b, scalars, m, n, sparseSafe, i * blklen, Math.min((i + 1) * blklen, m)));
            // execute tasks
            List<Future<Double>> taskret = pool.invokeAll(tasks);
            pool.shutdown();
            // aggregate partial results
            ValueFunction vfun = getAggFunction();
            if (vfun instanceof KahanFunction) {
                KahanObject kbuff = new KahanObject(0, 0);
                KahanPlus kplus = KahanPlus.getKahanPlusFnObject();
                for (Future<Double> task : taskret) kplus.execute2(kbuff, task.get());
                ret = kbuff._sum;
            } else {
                for (Future<Double> task : taskret) ret = vfun.execute(ret, task.get());
            }
        } catch (Exception ex) {
            throw new DMLRuntimeException(ex);
        }
    }
    // correction for min/max
    if ((_aggOp == AggOp.MIN || _aggOp == AggOp.MAX) && sparseSafe && a.getNonZeros() < a.getNumRows() * a.getNumColumns())
        // unseen 0 might be max or min value
        ret = getAggFunction().execute(ret, 0);
    return new DoubleObject(ret);
}

Example 35 with CompressedMatrixBlock

use of org.apache.sysml.runtime.compress.CompressedMatrixBlock in project incubator-systemml by apache.

the class SpoofCellwise method execute.

@Override
public MatrixBlock execute(ArrayList<MatrixBlock> inputs, ArrayList<ScalarObject> scalarObjects, MatrixBlock out, int k) {
    // sanity check
    if (inputs == null || inputs.size() < 1 || out == null)
        throw new RuntimeException("Invalid input arguments.");
    // input preparation
    MatrixBlock a = inputs.get(0);
    SideInput[] b = prepInputMatrices(inputs);
    double[] scalars = prepInputScalars(scalarObjects);
    final int m = a.getNumRows();
    final int n = a.getNumColumns();
    // sparse safe check
    boolean sparseSafe = isSparseSafe() || (b.length == 0 && genexec(0, b, scalars, m, n, 0, 0) == 0);
    long inputSize = sparseSafe ? getTotalInputNnz(inputs) : getTotalInputSize(inputs);
    if (inputSize < PAR_NUMCELL_THRESHOLD) {
        // serial execution
        k = 1;
    }
    // result allocation and preparations
    boolean sparseOut = _type == CellType.NO_AGG && sparseSafe && a.isInSparseFormat();
    switch(_type) {
        case NO_AGG:
            out.reset(m, n, sparseOut);
            break;
        case ROW_AGG:
            out.reset(m, 1, false);
            break;
        case COL_AGG:
            out.reset(1, n, false);
            break;
        default:
            throw new DMLRuntimeException("Invalid cell type: " + _type);
    }
    out.allocateBlock();
    long lnnz = 0;
    if (// SINGLE-THREADED
    k <= 1) {
        if (inputs.get(0) instanceof CompressedMatrixBlock)
            lnnz = executeCompressed((CompressedMatrixBlock) a, b, scalars, out, m, n, sparseSafe, 0, m);
        else if (!inputs.get(0).isInSparseFormat())
            lnnz = executeDense(a.getDenseBlock(), b, scalars, out, m, n, sparseSafe, 0, m);
        else
            lnnz = executeSparse(a.getSparseBlock(), b, scalars, out, m, n, sparseSafe, 0, m);
    } else // MULTI-THREADED
    {
        try {
            ExecutorService pool = CommonThreadPool.get(k);
            ArrayList<ParExecTask> tasks = new ArrayList<>();
            int nk = UtilFunctions.roundToNext(Math.min(8 * k, m / 32), k);
            int blklen = (int) (Math.ceil((double) m / nk));
            if (a instanceof CompressedMatrixBlock)
                blklen = BitmapEncoder.getAlignedBlocksize(blklen);
            for (int i = 0; i < nk & i * blklen < m; i++) tasks.add(new ParExecTask(a, b, scalars, out, m, n, sparseSafe, i * blklen, Math.min((i + 1) * blklen, m)));
            // execute tasks
            List<Future<Long>> taskret = pool.invokeAll(tasks);
            pool.shutdown();
            // aggregate nnz and error handling
            for (Future<Long> task : taskret) lnnz += task.get();
            if (_type == CellType.COL_AGG) {
                // aggregate partial results
                double[] c = out.getDenseBlockValues();
                ValueFunction vfun = getAggFunction();
                if (vfun instanceof KahanFunction) {
                    for (ParExecTask task : tasks) LibMatrixMult.vectAdd(task.getResult().getDenseBlockValues(), c, 0, 0, n);
                } else {
                    for (ParExecTask task : tasks) {
                        double[] tmp = task.getResult().getDenseBlockValues();
                        for (int j = 0; j < n; j++) c[j] = vfun.execute(c[j], tmp[j]);
                    }
                }
                lnnz = out.recomputeNonZeros();
            }
        } catch (Exception ex) {
            throw new DMLRuntimeException(ex);
        }
    }
    // post-processing
    out.setNonZeros(lnnz);
    out.examSparsity();
    return out;
}