Examples with CM_COV_Object org.apache.sysml.runtime.instructions.cp.CM_COV_Object used on opensource projects

Example 26 with CM_COV_Object

use of org.apache.sysml.runtime.instructions.cp.CM_COV_Object in project systemml by apache.

the class COV method execute.

@Override
public Data execute(Data in1, Data in2) {
    CM_COV_Object cov1 = (CM_COV_Object) in1;
    CM_COV_Object cov2 = (CM_COV_Object) in2;
    if (cov1.isCOVAllZeros()) {
        cov1.w = cov2.w;
        cov1.mean.set(cov2.mean);
        cov1.mean_v.set(cov2.mean_v);
        cov1.c2.set(cov2.c2);
        return cov1;
    }
    if (cov2.isCOVAllZeros())
        return cov1;
    double w = cov1.w + cov2.w;
    double du = cov2.mean._sum - cov1.mean._sum;
    double dv = cov2.mean_v._sum - cov1.mean_v._sum;
    cov1.mean = (KahanObject) _plus.execute(cov1.mean, cov2.w * du / w);
    cov1.mean_v = (KahanObject) _plus.execute(cov1.mean_v, cov2.w * dv / w);
    cov1.c2 = (KahanObject) _plus.execute(cov1.c2, cov2.c2._sum, cov2.c2._correction);
    cov1.c2 = (KahanObject) _plus.execute(cov1.c2, cov1.w * cov2.w / w * du * dv);
    cov1.w = w;
    return cov1;
}

Example 27 with CM_COV_Object

use of org.apache.sysml.runtime.instructions.cp.CM_COV_Object in project systemml by apache.

the class CM method execute.

/**
 * Combining stats from two partitions of the data.
 */
@Override
public Data execute(Data in1, Data in2) {
    CM_COV_Object cm1 = (CM_COV_Object) in1;
    CM_COV_Object cm2 = (CM_COV_Object) in2;
    if (cm1.isCMAllZeros()) {
        cm1.w = cm2.w;
        cm1.mean.set(cm2.mean);
        cm1.m2.set(cm2.m2);
        cm1.m3.set(cm2.m3);
        cm1.m4.set(cm2.m4);
        return cm1;
    }
    if (cm2.isCMAllZeros())
        return cm1;
    switch(_type) {
        case COUNT:
            {
                cm1.w = Math.round(cm1.w + cm2.w);
                break;
            }
        case MEAN:
            {
                double w = cm1.w + cm2.w;
                double d = cm2.mean._sum - cm1.mean._sum;
                cm1.mean = (KahanObject) _plus.execute(cm1.mean, cm2.w * d / w);
                cm1.w = w;
                break;
            }
        case CM2:
            {
                double w = cm1.w + cm2.w;
                double d = cm2.mean._sum - cm1.mean._sum;
                cm1.mean = (KahanObject) _plus.execute(cm1.mean, cm2.w * d / w);
                double t1 = cm1.w * cm2.w / w * d;
                double lt1 = t1 * d;
                _buff2.set(cm1.m2);
                _buff2 = (KahanObject) _plus.execute(_buff2, cm2.m2._sum, cm2.m2._correction);
                _buff2 = (KahanObject) _plus.execute(_buff2, lt1);
                cm1.m2.set(_buff2);
                cm1.w = w;
                break;
            }
        case CM3:
            {
                double w = cm1.w + cm2.w;
                double d = cm2.mean._sum - cm1.mean._sum;
                cm1.mean = (KahanObject) _plus.execute(cm1.mean, cm2.w * d / w);
                double t1 = cm1.w * cm2.w / w * d;
                double t2 = -1 / cm1.w;
                double lt1 = t1 * d;
                double lt2 = Math.pow(t1, 3) * (1 / Math.pow(cm2.w, 2) - Math.pow(t2, 2));
                double f1 = cm1.w / w;
                double f2 = cm2.w / w;
                _buff2.set(cm1.m2);
                _buff2 = (KahanObject) _plus.execute(_buff2, cm2.m2._sum, cm2.m2._correction);
                _buff2 = (KahanObject) _plus.execute(_buff2, lt1);
                _buff3.set(cm1.m3);
                _buff3 = (KahanObject) _plus.execute(_buff3, cm2.m3._sum, cm2.m3._correction);
                _buff3 = (KahanObject) _plus.execute(_buff3, 3 * (-f2 * cm1.m2._sum + f1 * cm2.m2._sum) * d + lt2);
                cm1.m2.set(_buff2);
                cm1.m3.set(_buff3);
                cm1.w = w;
                break;
            }
        case CM4:
            {
                double w = cm1.w + cm2.w;
                double d = cm2.mean._sum - cm1.mean._sum;
                cm1.mean = (KahanObject) _plus.execute(cm1.mean, cm2.w * d / w);
                double t1 = cm1.w * cm2.w / w * d;
                double t2 = -1 / cm1.w;
                double lt1 = t1 * d;
                double lt2 = Math.pow(t1, 3) * (1 / Math.pow(cm2.w, 2) - Math.pow(t2, 2));
                double lt3 = Math.pow(t1, 4) * (1 / Math.pow(cm2.w, 3) - Math.pow(t2, 3));
                double f1 = cm1.w / w;
                double f2 = cm2.w / w;
                _buff2.set(cm1.m2);
                _buff2 = (KahanObject) _plus.execute(_buff2, cm2.m2._sum, cm2.m2._correction);
                _buff2 = (KahanObject) _plus.execute(_buff2, lt1);
                _buff3.set(cm1.m3);
                _buff3 = (KahanObject) _plus.execute(_buff3, cm2.m3._sum, cm2.m3._correction);
                _buff3 = (KahanObject) _plus.execute(_buff3, 3 * (-f2 * cm1.m2._sum + f1 * cm2.m2._sum) * d + lt2);
                cm1.m4 = (KahanObject) _plus.execute(cm1.m4, cm2.m4._sum, cm2.m4._correction);
                cm1.m4 = (KahanObject) _plus.execute(cm1.m4, 4 * (-f2 * cm1.m3._sum + f1 * cm2.m3._sum) * d + 6 * (Math.pow(-f2, 2) * cm1.m2._sum + Math.pow(f1, 2) * cm2.m2._sum) * Math.pow(d, 2) + lt3);
                cm1.m2.set(_buff2);
                cm1.m3.set(_buff3);
                cm1.w = w;
                break;
            }
        case VARIANCE:
            {
                double w = cm1.w + cm2.w;
                double d = cm2.mean._sum - cm1.mean._sum;
                cm1.mean = (KahanObject) _plus.execute(cm1.mean, cm2.w * d / w);
                double t1 = cm1.w * cm2.w / w * d;
                double lt1 = t1 * d;
                cm1.m2 = (KahanObject) _plus.execute(cm1.m2, cm2.m2._sum, cm2.m2._correction);
                cm1.m2 = (KahanObject) _plus.execute(cm1.m2, lt1);
                cm1.w = w;
                break;
            }
        default:
            throw new DMLRuntimeException("Unsupported operation type: " + _type);
    }
    return cm1;
}

Example 28 with CM_COV_Object

use of org.apache.sysml.runtime.instructions.cp.CM_COV_Object in project systemml by apache.

the class CMCOVMRReducer method configure.

@Override
public void configure(JobConf job) {
    super.configure(job);
    try {
        cmNcovInstructions = MRJobConfiguration.getCM_N_COVInstructions(job);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    rlens = new HashMap<>();
    clens = new HashMap<>();
    for (CM_N_COVInstruction ins : cmNcovInstructions) {
        if (ins.getOperator() instanceof COVOperator)
            covTags.add(ins.input);
        else
            // CMOperator
            cmFn.put(ins.input, CM.getCMFnObject(((CMOperator) ins.getOperator()).getAggOpType()));
        outputIndexesMapping.put(ins.output, getOutputIndexes(ins.output));
        rlens.put(ins.input, MRJobConfiguration.getNumRows(job, ins.input));
        clens.put(ins.input, MRJobConfiguration.getNumColumns(job, ins.input));
    }
    zeroObj = new CM_COV_Object();
    zeroObj.w = 1;
}

Example 29 with CM_COV_Object

use of org.apache.sysml.runtime.instructions.cp.CM_COV_Object in project systemml by apache.

the class LibMatrixAgg method groupedAggregateCM.

private static void groupedAggregateCM(MatrixBlock groups, MatrixBlock target, MatrixBlock weights, MatrixBlock result, int numGroups, CMOperator cmOp, int cl, int cu) {
    CM cmFn = CM.getCMFnObject(((CMOperator) cmOp).getAggOpType());
    // default weight
    double w = 1;
    // init group buffers
    int numCols2 = cu - cl;
    CM_COV_Object[][] cmValues = new CM_COV_Object[numGroups][numCols2];
    for (int i = 0; i < numGroups; i++) for (int j = 0; j < numCols2; j++) cmValues[i][j] = new CM_COV_Object();
    // column vector or matrix
    if (// SPARSE target
    target.sparse) {
        SparseBlock a = target.sparseBlock;
        for (int i = 0; i < groups.getNumRows(); i++) {
            int g = (int) groups.quickGetValue(i, 0);
            if (g > numGroups)
                continue;
            if (!a.isEmpty(i)) {
                int pos = a.pos(i);
                int len = a.size(i);
                int[] aix = a.indexes(i);
                double[] avals = a.values(i);
                int j = (cl == 0) ? 0 : a.posFIndexGTE(i, cl);
                j = (j >= 0) ? pos + j : pos + len;
                for (; // for each nnz
                j < pos + len && aix[j] < cu; // for each nnz
                j++) {
                    if (weights != null)
                        w = weights.quickGetValue(i, 0);
                    cmFn.execute(cmValues[g - 1][aix[j] - cl], avals[j], w);
                }
            // TODO sparse unsafe correction
            }
        }
    } else // DENSE target
    {
        DenseBlock a = target.getDenseBlock();
        for (int i = 0; i < groups.getNumRows(); i++) {
            int g = (int) groups.quickGetValue(i, 0);
            if (g > numGroups)
                continue;
            double[] avals = a.values(i);
            int aix = a.pos(i);
            for (int j = cl; j < cu; j++) {
                // sparse unsafe
                double d = avals[aix + j];
                if (weights != null)
                    w = weights.quickGetValue(i, 0);
                // buffer is 0-indexed, whereas range of values for g = [1,numGroups]
                cmFn.execute(cmValues[g - 1][j - cl], d, w);
            }
        }
    }
    // extract the required value from each CM_COV_Object
    for (int i = 0; i < numGroups; i++) for (int j = 0; j < numCols2; j++) {
        // result is 0-indexed, so is cmValues
        result.appendValue(i, j, cmValues[i][j + cl].getRequiredResult(cmOp));
    }
}

Example 30 with CM_COV_Object

use of org.apache.sysml.runtime.instructions.cp.CM_COV_Object in project systemml by apache.

the class LibMatrixAgg method aggregateUnaryMatrixSparse.

private static void aggregateUnaryMatrixSparse(MatrixBlock in, MatrixBlock out, AggType optype, ValueFunction vFn, IndexFunction ixFn, int rl, int ru) {
    final int m = in.rlen;
    final int n = in.clen;
    // note: due to corrections, even the output might be a large dense block
    SparseBlock a = in.getSparseBlock();
    DenseBlock c = out.getDenseBlock();
    switch(optype) {
        case KAHAN_SUM:
            {
                // SUM via k+
                KahanObject kbuff = new KahanObject(0, 0);
                if (// SUM
                ixFn instanceof ReduceAll)
                    s_uakp(a, c, n, kbuff, (KahanPlus) vFn, rl, ru);
                else if (// ROWSUM
                ixFn instanceof ReduceCol)
                    s_uarkp(a, c, n, kbuff, (KahanPlus) vFn, rl, ru);
                else if (// COLSUM
                ixFn instanceof ReduceRow)
                    s_uackp(a, c, n, kbuff, (KahanPlus) vFn, rl, ru);
                else if (// TRACE
                ixFn instanceof ReduceDiag)
                    s_uakptrace(a, c, n, kbuff, (KahanPlus) vFn, rl, ru);
                break;
            }
        case KAHAN_SUM_SQ:
            {
                // SUM_SQ via k+
                KahanObject kbuff = new KahanObject(0, 0);
                if (// SUM_SQ
                ixFn instanceof ReduceAll)
                    s_uasqkp(a, c, n, kbuff, (KahanPlusSq) vFn, rl, ru);
                else if (// ROWSUM_SQ
                ixFn instanceof ReduceCol)
                    s_uarsqkp(a, c, n, kbuff, (KahanPlusSq) vFn, rl, ru);
                else if (// COLSUM_SQ
                ixFn instanceof ReduceRow)
                    s_uacsqkp(a, c, n, kbuff, (KahanPlusSq) vFn, rl, ru);
                break;
            }
        case CUM_KAHAN_SUM:
            {
                // CUMSUM
                KahanObject kbuff = new KahanObject(0, 0);
                KahanPlus kplus = KahanPlus.getKahanPlusFnObject();
                s_ucumkp(a, null, out.getDenseBlock(), m, n, kbuff, kplus, rl, ru);
                break;
            }
        case CUM_PROD:
            {
                // CUMPROD
                s_ucumm(a, null, out.getDenseBlockValues(), n, rl, ru);
                break;
            }
        case CUM_MIN:
        case CUM_MAX:
            {
                double init = (optype == AggType.CUM_MAX) ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY;
                s_ucummxx(a, null, out.getDenseBlockValues(), n, init, (Builtin) vFn, rl, ru);
                break;
            }
        case MIN:
        case MAX:
            {
                // MAX/MIN
                double init = (optype == AggType.MAX) ? Double.NEGATIVE_INFINITY : Double.POSITIVE_INFINITY;
                if (// MIN/MAX
                ixFn instanceof ReduceAll)
                    s_uamxx(a, c, n, init, (Builtin) vFn, rl, ru);
                else if (// ROWMIN/ROWMAX
                ixFn instanceof ReduceCol)
                    s_uarmxx(a, c, n, init, (Builtin) vFn, rl, ru);
                else if (// COLMIN/COLMAX
                ixFn instanceof ReduceRow)
                    s_uacmxx(a, c, m, n, init, (Builtin) vFn, rl, ru);
                break;
            }
        case MAX_INDEX:
            {
                double init = Double.NEGATIVE_INFINITY;
                if (// ROWINDEXMAX
                ixFn instanceof ReduceCol)
                    s_uarimxx(a, c, n, init, (Builtin) vFn, rl, ru);
                break;
            }
        case MIN_INDEX:
            {
                double init = Double.POSITIVE_INFINITY;
                if (// ROWINDEXMAX
                ixFn instanceof ReduceCol)
                    s_uarimin(a, c, n, init, (Builtin) vFn, rl, ru);
                break;
            }
        case MEAN:
            {
                KahanObject kbuff = new KahanObject(0, 0);
                if (// MEAN
                ixFn instanceof ReduceAll)
                    s_uamean(a, c, n, kbuff, (Mean) vFn, rl, ru);
                else if (// ROWMEAN
                ixFn instanceof ReduceCol)
                    s_uarmean(a, c, n, kbuff, (Mean) vFn, rl, ru);
                else if (// COLMEAN
                ixFn instanceof ReduceRow)
                    s_uacmean(a, c, n, kbuff, (Mean) vFn, rl, ru);
                break;
            }
        case VAR:
            {
                // VAR
                CM_COV_Object cbuff = new CM_COV_Object();
                if (// VAR
                ixFn instanceof ReduceAll)
                    s_uavar(a, c, n, cbuff, (CM) vFn, rl, ru);
                else if (// ROWVAR
                ixFn instanceof ReduceCol)
                    s_uarvar(a, c, n, cbuff, (CM) vFn, rl, ru);
                else if (// COLVAR
                ixFn instanceof ReduceRow)
                    s_uacvar(a, c, n, cbuff, (CM) vFn, rl, ru);
                break;
            }
        case PROD:
            {
                // PROD
                if (// PROD
                ixFn instanceof ReduceAll)
                    s_uam(a, c, n, rl, ru);
                break;
            }
        default:
            throw new DMLRuntimeException("Unsupported aggregation type: " + optype);
    }
}