Examples with CM org.apache.sysml.runtime.functionobjects.CM used on opensource projects

Example 6 with CM

use of org.apache.sysml.runtime.functionobjects.CM in project incubator-systemml by apache.

the class LibMatrixAgg method groupedAggregateCM.

private static void groupedAggregateCM(MatrixBlock groups, MatrixBlock target, MatrixBlock weights, MatrixBlock result, int numGroups, CMOperator cmOp, int cl, int cu) throws DMLRuntimeException {
    CM cmFn = CM.getCMFnObject(((CMOperator) cmOp).getAggOpType());
    //default weight
    double w = 1;
    //init group buffers
    int numCols2 = cu - cl;
    CM_COV_Object[][] cmValues = new CM_COV_Object[numGroups][numCols2];
    for (int i = 0; i < numGroups; i++) for (int j = 0; j < numCols2; j++) cmValues[i][j] = new CM_COV_Object();
    //column vector or matrix
    if (//SPARSE target
    target.sparse) {
        SparseBlock a = target.sparseBlock;
        for (int i = 0; i < groups.getNumRows(); i++) {
            int g = (int) groups.quickGetValue(i, 0);
            if (g > numGroups)
                continue;
            if (!a.isEmpty(i)) {
                int pos = a.pos(i);
                int len = a.size(i);
                int[] aix = a.indexes(i);
                double[] avals = a.values(i);
                int j = (cl == 0) ? pos : a.posFIndexGTE(i, cl);
                j = (j >= 0) ? j : pos + len;
                for (; //for each nnz
                j < pos + len && aix[j] < cu; //for each nnz
                j++) {
                    if (weights != null)
                        w = weights.quickGetValue(i, 0);
                    cmFn.execute(cmValues[g - 1][aix[j] - cl], avals[j], w);
                }
            //TODO sparse unsafe correction
            }
        }
    } else //DENSE target
    {
        double[] a = target.denseBlock;
        for (int i = 0, aix = 0; i < groups.getNumRows(); i++, aix += target.clen) {
            int g = (int) groups.quickGetValue(i, 0);
            if (g > numGroups)
                continue;
            for (int j = cl; j < cu; j++) {
                //sparse unsafe
                double d = a[aix + j];
                if (weights != null)
                    w = weights.quickGetValue(i, 0);
                // buffer is 0-indexed, whereas range of values for g = [1,numGroups]
                cmFn.execute(cmValues[g - 1][j - cl], d, w);
            }
        }
    }
    // extract the required value from each CM_COV_Object
    for (int i = 0; i < numGroups; i++) for (int j = 0; j < numCols2; j++) {
        // result is 0-indexed, so is cmValues
        result.appendValue(i, j, cmValues[i][j + cl].getRequiredResult(cmOp));
    }
}

Example 7 with CM

use of org.apache.sysml.runtime.functionobjects.CM in project incubator-systemml by apache.

the class GroupedAggMRCombiner method reduce.

@Override
public void reduce(TaggedMatrixIndexes key, Iterator<WeightedCell> values, OutputCollector<TaggedMatrixIndexes, WeightedCell> out, Reporter reporter) throws IOException {
    long start = System.currentTimeMillis();
    //get aggregate operator
    GroupedAggregateInstruction ins = grpaggInstructions.get(key.getTag());
    Operator op = ins.getOperator();
    boolean isPartialAgg = true;
    //combine iterator to single value
    try {
        if (//everything except sum
        op instanceof CMOperator) {
            if (((CMOperator) op).isPartialAggregateOperator()) {
                cmObj.reset();
                CM lcmFn = cmFn.get(key.getTag());
                //partial aggregate cm operator 
                while (values.hasNext()) {
                    WeightedCell value = values.next();
                    lcmFn.execute(cmObj, value.getValue(), value.getWeight());
                }
                outCell.setValue(cmObj.getRequiredPartialResult(op));
                outCell.setWeight(cmObj.getWeight());
            } else //forward tuples to reducer
            {
                isPartialAgg = false;
                while (values.hasNext()) out.collect(key, values.next());
            }
        } else if (//sum
        op instanceof AggregateOperator) {
            AggregateOperator aggop = (AggregateOperator) op;
            if (aggop.correctionExists) {
                KahanObject buffer = new KahanObject(aggop.initialValue, 0);
                KahanPlus.getKahanPlusFnObject();
                //partial aggregate with correction
                while (values.hasNext()) {
                    WeightedCell value = values.next();
                    aggop.increOp.fn.execute(buffer, value.getValue() * value.getWeight());
                }
                outCell.setValue(buffer._sum);
                outCell.setWeight(1);
            } else //no correction
            {
                double v = aggop.initialValue;
                //partial aggregate without correction
                while (values.hasNext()) {
                    WeightedCell value = values.next();
                    v = aggop.increOp.fn.execute(v, value.getValue() * value.getWeight());
                }
                outCell.setValue(v);
                outCell.setWeight(1);
            }
        } else
            throw new IOException("Unsupported operator in instruction: " + ins);
    } catch (Exception ex) {
        throw new IOException(ex);
    }
    //collect the output (to reducer)
    if (isPartialAgg)
        out.collect(key, outCell);
    reporter.incrCounter(Counters.COMBINE_OR_REDUCE_TIME, System.currentTimeMillis() - start);
}

Example 8 with CM

use of org.apache.sysml.runtime.functionobjects.CM in project incubator-systemml by apache.

the class GroupedAggMRReducer method reduce.

@Override
public void reduce(TaggedMatrixIndexes key, Iterator<WeightedCell> values, OutputCollector<MatrixIndexes, MatrixCell> out, Reporter report) throws IOException {
    commonSetup(report);
    //get operator
    GroupedAggregateInstruction ins = grpaggInstructions.get(key.getTag());
    Operator op = ins.getOperator();
    try {
        if (//all, but sum
        op instanceof CMOperator) {
            cmObj.reset();
            CM lcmFn = cmFn.get(key.getTag());
            while (values.hasNext()) {
                WeightedCell value = values.next();
                lcmFn.execute(cmObj, value.getValue(), value.getWeight());
            }
            outCell.setValue(cmObj.getRequiredResult(op));
        } else if (//sum
        op instanceof AggregateOperator) {
            AggregateOperator aggop = (AggregateOperator) op;
            if (aggop.correctionExists) {
                KahanObject buffer = new KahanObject(aggop.initialValue, 0);
                while (values.hasNext()) {
                    WeightedCell value = values.next();
                    aggop.increOp.fn.execute(buffer, value.getValue() * value.getWeight());
                }
                outCell.setValue(buffer._sum);
            } else {
                double v = aggop.initialValue;
                while (values.hasNext()) {
                    WeightedCell value = values.next();
                    v = aggop.increOp.fn.execute(v, value.getValue() * value.getWeight());
                }
                outCell.setValue(v);
            }
        } else
            throw new IOException("Unsupported operator in instruction: " + ins);
    } catch (Exception ex) {
        throw new IOException(ex);
    }
    outIndex.setIndexes(key.getBaseObject());
    cachedValues.reset();
    cachedValues.set(key.getTag(), outIndex, outCell);
    processReducerInstructions();
    //output the final result matrices
    outputResultsFromCachedValues(report);
}

Example 9 with CM

use of org.apache.sysml.runtime.functionobjects.CM in project incubator-systemml by apache.

the class CMCOVMRMapper method specialOperationsForActualMap.

@Override
protected void specialOperationsForActualMap(int index, OutputCollector<Writable, Writable> out, Reporter reporter) throws IOException {
    //apply all instructions
    processMapperInstructionsForMatrix(index);
    for (byte tag : cmTags) {
        CM lcmFn = cmFn.get(tag);
        IndexedMatrixValue input = cachedValues.getFirst(tag);
        if (input == null)
            continue;
        WeightedPair inputPair = (WeightedPair) input.getValue();
        CM_N_COVCell cmValue = (CM_N_COVCell) cmNcovCache.getFirst(tag).getValue();
        try {
            //	System.out.println("~~~~~\nold: "+cmValue.getCM_N_COVObject());
            //	System.out.println("add: "+inputPair);
            lcmFn.execute(cmValue.getCM_N_COVObject(), inputPair.getValue(), inputPair.getWeight());
        //	System.out.println("new: "+cmValue.getCM_N_COVObject());
        } catch (DMLRuntimeException e) {
            throw new IOException(e);
        }
    }
    for (byte tag : covTags) {
        IndexedMatrixValue input = cachedValues.getFirst(tag);
        if (input == null)
            continue;
        //System.out.println("*** cached Value:\n"+cachedValues);
        WeightedPair inputPair = (WeightedPair) input.getValue();
        CM_N_COVCell comValue = (CM_N_COVCell) cmNcovCache.getFirst(tag).getValue();
        try {
            //System.out.println("~~~~~\nold: "+comValue.getCM_N_COVObject());
            //	System.out.println("add: "+inputPair);
            covFn.execute(comValue.getCM_N_COVObject(), inputPair.getValue(), inputPair.getOtherValue(), inputPair.getWeight());
        //	System.out.println("new: "+comValue.getCM_N_COVObject());
        } catch (DMLRuntimeException e) {
            throw new IOException(e);
        }
    }
}

Example 10 with CM

use of org.apache.sysml.runtime.functionobjects.CM in project incubator-systemml by apache.

the class PerformGroupByAggInCombiner method call.

@Override
public WeightedCell call(WeightedCell value1, WeightedCell value2) throws Exception {
    WeightedCell outCell = new WeightedCell();
    CM_COV_Object cmObj = new CM_COV_Object();
    if (//everything except sum
    _op instanceof CMOperator) {
        if (((CMOperator) _op).isPartialAggregateOperator()) {
            cmObj.reset();
            // cmFn.get(key.getTag());
            CM lcmFn = CM.getCMFnObject(((CMOperator) _op).aggOpType);
            //partial aggregate cm operator
            lcmFn.execute(cmObj, value1.getValue(), value1.getWeight());
            lcmFn.execute(cmObj, value2.getValue(), value2.getWeight());
            outCell.setValue(cmObj.getRequiredPartialResult(_op));
            outCell.setWeight(cmObj.getWeight());
        } else //forward tuples to reducer
        {
            throw new DMLRuntimeException("Incorrect usage, should have used PerformGroupByAggInReducer");
        }
    } else if (//sum
    _op instanceof AggregateOperator) {
        AggregateOperator aggop = (AggregateOperator) _op;
        if (aggop.correctionExists) {
            KahanObject buffer = new KahanObject(aggop.initialValue, 0);
            KahanPlus.getKahanPlusFnObject();
            //partial aggregate with correction
            aggop.increOp.fn.execute(buffer, value1.getValue() * value1.getWeight());
            aggop.increOp.fn.execute(buffer, value2.getValue() * value2.getWeight());
            outCell.setValue(buffer._sum);
            outCell.setWeight(1);
        } else //no correction
        {
            double v = aggop.initialValue;
            //partial aggregate without correction
            v = aggop.increOp.fn.execute(v, value1.getValue() * value1.getWeight());
            v = aggop.increOp.fn.execute(v, value2.getValue() * value2.getWeight());
            outCell.setValue(v);
            outCell.setWeight(1);
        }
    } else
        throw new DMLRuntimeException("Unsupported operator in grouped aggregate instruction:" + _op);
    return outCell;
}