Examples with CMOperator org.apache.sysml.runtime.matrix.operators.CMOperator used on opensource projects

Example 6 with CMOperator

use of org.apache.sysml.runtime.matrix.operators.CMOperator in project incubator-systemml by apache.

the class CentralMomentSPInstruction method processInstruction.

@Override
public void processInstruction(ExecutionContext ec) {
    SparkExecutionContext sec = (SparkExecutionContext) ec;
    // parse 'order' input argument
    CPOperand scalarInput = (input3 == null ? input2 : input3);
    ScalarObject order = ec.getScalarInput(scalarInput.getName(), scalarInput.getValueType(), scalarInput.isLiteral());
    CMOperator cop = ((CMOperator) _optr);
    if (cop.getAggOpType() == AggregateOperationTypes.INVALID) {
        cop.setCMAggOp((int) order.getLongValue());
    }
    // get input
    JavaPairRDD<MatrixIndexes, MatrixBlock> in1 = sec.getBinaryBlockRDDHandleForVariable(input1.getName());
    // process central moment instruction
    CM_COV_Object cmobj = null;
    if (// w/o weights
    input3 == null) {
        cmobj = in1.values().map(new RDDCMFunction(cop)).fold(new CM_COV_Object(), new RDDCMReduceFunction(cop));
    } else // with weights
    {
        JavaPairRDD<MatrixIndexes, MatrixBlock> in2 = sec.getBinaryBlockRDDHandleForVariable(input2.getName());
        cmobj = in1.join(in2).values().map(new RDDCMWeightsFunction(cop)).fold(new CM_COV_Object(), new RDDCMReduceFunction(cop));
    }
    // create scalar output (no lineage information required)
    double val = cmobj.getRequiredResult(_optr);
    ec.setScalarOutput(output.getName(), new DoubleObject(val));
}

Example 7 with CMOperator

use of org.apache.sysml.runtime.matrix.operators.CMOperator in project incubator-systemml by apache.

the class LibMatrixAgg method groupedAggregate.

public static void groupedAggregate(MatrixBlock groups, MatrixBlock target, MatrixBlock weights, MatrixBlock result, int numGroups, Operator op, int k) {
    // fall back to sequential version if necessary
    boolean rowVector = (target.getNumRows() == 1 && target.getNumColumns() > 1);
    if (k <= 1 || (long) target.rlen * target.clen < PAR_NUMCELL_THRESHOLD || rowVector || target.clen == 1) {
        groupedAggregate(groups, target, weights, result, numGroups, op);
        return;
    }
    if (!(op instanceof CMOperator || op instanceof AggregateOperator)) {
        throw new DMLRuntimeException("Invalid operator (" + op + ") encountered while processing groupedAggregate.");
    }
    // preprocessing (no need to check isThreadSafe)
    result.sparse = false;
    result.allocateDenseBlock();
    // (currently: parallelization over columns to avoid additional memory requirements)
    try {
        ExecutorService pool = CommonThreadPool.get(k);
        ArrayList<GrpAggTask> tasks = new ArrayList<>();
        int blklen = (int) (Math.ceil((double) target.clen / k));
        for (int i = 0; i < k & i * blklen < target.clen; i++) tasks.add(new GrpAggTask(groups, target, weights, result, numGroups, op, i * blklen, Math.min((i + 1) * blklen, target.clen)));
        List<Future<Object>> taskret = pool.invokeAll(tasks);
        pool.shutdown();
        for (Future<Object> task : taskret) // error handling
        task.get();
    } catch (Exception ex) {
        throw new DMLRuntimeException(ex);
    }
    // postprocessing
    result.recomputeNonZeros();
    result.examSparsity();
}

Example 8 with CMOperator

use of org.apache.sysml.runtime.matrix.operators.CMOperator in project incubator-systemml by apache.

the class GroupedAggMRCombiner method reduce.

@Override
public void reduce(TaggedMatrixIndexes key, Iterator<WeightedCell> values, OutputCollector<TaggedMatrixIndexes, WeightedCell> out, Reporter reporter) throws IOException {
    long start = System.currentTimeMillis();
    // get aggregate operator
    GroupedAggregateInstruction ins = grpaggInstructions.get(key.getTag());
    Operator op = ins.getOperator();
    boolean isPartialAgg = true;
    // combine iterator to single value
    try {
        if (// everything except sum
        op instanceof CMOperator) {
            if (((CMOperator) op).isPartialAggregateOperator()) {
                cmObj.reset();
                CM lcmFn = cmFn.get(key.getTag());
                // partial aggregate cm operator
                while (values.hasNext()) {
                    WeightedCell value = values.next();
                    lcmFn.execute(cmObj, value.getValue(), value.getWeight());
                }
                outCell.setValue(cmObj.getRequiredPartialResult(op));
                outCell.setWeight(cmObj.getWeight());
            } else // forward tuples to reducer
            {
                isPartialAgg = false;
                while (values.hasNext()) out.collect(key, values.next());
            }
        } else if (// sum
        op instanceof AggregateOperator) {
            AggregateOperator aggop = (AggregateOperator) op;
            if (aggop.correctionExists) {
                KahanObject buffer = new KahanObject(aggop.initialValue, 0);
                KahanPlus.getKahanPlusFnObject();
                // partial aggregate with correction
                while (values.hasNext()) {
                    WeightedCell value = values.next();
                    aggop.increOp.fn.execute(buffer, value.getValue() * value.getWeight());
                }
                outCell.setValue(buffer._sum);
                outCell.setWeight(1);
            } else // no correction
            {
                double v = aggop.initialValue;
                // partial aggregate without correction
                while (values.hasNext()) {
                    WeightedCell value = values.next();
                    v = aggop.increOp.fn.execute(v, value.getValue() * value.getWeight());
                }
                outCell.setValue(v);
                outCell.setWeight(1);
            }
        } else
            throw new IOException("Unsupported operator in instruction: " + ins);
    } catch (Exception ex) {
        throw new IOException(ex);
    }
    // collect the output (to reducer)
    if (isPartialAgg)
        out.collect(key, outCell);
    reporter.incrCounter(Counters.COMBINE_OR_REDUCE_TIME, System.currentTimeMillis() - start);
}

Example 9 with CMOperator

use of org.apache.sysml.runtime.matrix.operators.CMOperator in project incubator-systemml by apache.

the class GroupedAggMRReducer method reduce.

@Override
public void reduce(TaggedMatrixIndexes key, Iterator<WeightedCell> values, OutputCollector<MatrixIndexes, MatrixCell> out, Reporter report) throws IOException {
    commonSetup(report);
    // get operator
    GroupedAggregateInstruction ins = grpaggInstructions.get(key.getTag());
    Operator op = ins.getOperator();
    try {
        if (// all, but sum
        op instanceof CMOperator) {
            cmObj.reset();
            CM lcmFn = cmFn.get(key.getTag());
            while (values.hasNext()) {
                WeightedCell value = values.next();
                lcmFn.execute(cmObj, value.getValue(), value.getWeight());
            }
            outCell.setValue(cmObj.getRequiredResult(op));
        } else if (// sum
        op instanceof AggregateOperator) {
            AggregateOperator aggop = (AggregateOperator) op;
            if (aggop.correctionExists) {
                KahanObject buffer = new KahanObject(aggop.initialValue, 0);
                while (values.hasNext()) {
                    WeightedCell value = values.next();
                    aggop.increOp.fn.execute(buffer, value.getValue() * value.getWeight());
                }
                outCell.setValue(buffer._sum);
            } else {
                double v = aggop.initialValue;
                while (values.hasNext()) {
                    WeightedCell value = values.next();
                    v = aggop.increOp.fn.execute(v, value.getValue() * value.getWeight());
                }
                outCell.setValue(v);
            }
        } else
            throw new IOException("Unsupported operator in instruction: " + ins);
    } catch (Exception ex) {
        throw new IOException(ex);
    }
    outIndex.setIndexes(key.getBaseObject());
    cachedValues.reset();
    cachedValues.set(key.getTag(), outIndex, outCell);
    processReducerInstructions();
    // output the final result matrices
    outputResultsFromCachedValues(report);
}

Example 10 with CMOperator

use of org.apache.sysml.runtime.matrix.operators.CMOperator in project incubator-systemml by apache.

the class BasicMatrixCentralMomentTest method runMatrixAppendTest.

private static void runMatrixAppendTest(SparsityType sptype, ValueType vtype, boolean compress) {
    try {
        // prepare sparsity for input data
        double sparsity = -1;
        switch(sptype) {
            case DENSE:
                sparsity = sparsity1;
                break;
            case SPARSE:
                sparsity = sparsity2;
                break;
            case EMPTY:
                sparsity = sparsity3;
                break;
        }
        // generate input data
        double min = (vtype == ValueType.CONST) ? 10 : -10;
        double[][] input = TestUtils.generateTestMatrix(rows, cols, min, 10, sparsity, 7);
        if (vtype == ValueType.RAND_ROUND_OLE || vtype == ValueType.RAND_ROUND_DDC) {
            CompressedMatrixBlock.ALLOW_DDC_ENCODING = (vtype == ValueType.RAND_ROUND_DDC);
            input = TestUtils.round(input);
        }
        MatrixBlock mb = DataConverter.convertToMatrixBlock(input);
        // compress given matrix block
        CompressedMatrixBlock cmb = new CompressedMatrixBlock(mb);
        if (compress)
            cmb.compress();
        // quantile uncompressed
        AggregateOperationTypes opType = CMOperator.getCMAggOpType(2);
        CMOperator cm = new CMOperator(CM.getCMFnObject(opType), opType);
        double ret1 = mb.cmOperations(cm).getRequiredResult(opType);
        // quantile compressed
        double ret2 = cmb.cmOperations(cm).getRequiredResult(opType);
        // compare result with input
        TestUtils.compareScalars(ret1, ret2, 0.0000001);
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    } finally {
        CompressedMatrixBlock.ALLOW_DDC_ENCODING = true;
    }
}