use of org.apache.sysml.runtime.functionobjects.CM in project incubator-systemml by apache.
the class PerformGroupByAggInReducer method call.
@Override
public WeightedCell call(Iterable<WeightedCell> kv) throws Exception {
WeightedCell outCell = new WeightedCell();
CM_COV_Object cmObj = new CM_COV_Object();
if (// everything except sum
op instanceof CMOperator) {
cmObj.reset();
// cmFn.get(key.getTag());
CM lcmFn = CM.getCMFnObject(((CMOperator) op).aggOpType);
if (((CMOperator) op).isPartialAggregateOperator()) {
throw new DMLRuntimeException("Incorrect usage, should have used PerformGroupByAggInCombiner");
} else // forward tuples to reducer
{
for (WeightedCell value : kv) lcmFn.execute(cmObj, value.getValue(), value.getWeight());
outCell.setValue(cmObj.getRequiredResult(op));
outCell.setWeight(1);
}
} else if (// sum
op instanceof AggregateOperator) {
AggregateOperator aggop = (AggregateOperator) op;
if (aggop.correctionExists) {
KahanObject buffer = new KahanObject(aggop.initialValue, 0);
KahanPlus.getKahanPlusFnObject();
// partial aggregate with correction
for (WeightedCell value : kv) aggop.increOp.fn.execute(buffer, value.getValue() * value.getWeight());
outCell.setValue(buffer._sum);
outCell.setWeight(1);
} else // no correction
{
double v = aggop.initialValue;
// partial aggregate without correction
for (WeightedCell value : kv) v = aggop.increOp.fn.execute(v, value.getValue() * value.getWeight());
outCell.setValue(v);
outCell.setWeight(1);
}
} else
throw new DMLRuntimeException("Unsupported operator in grouped aggregate instruction:" + op);
return outCell;
}
use of org.apache.sysml.runtime.functionobjects.CM in project incubator-systemml by apache.
the class LibMatrixAgg method getAggType.
private static AggType getAggType(AggregateUnaryOperator op) {
ValueFunction vfn = op.aggOp.increOp.fn;
IndexFunction ifn = op.indexFn;
// (kahan) sum / sum squared / trace (for ReduceDiag)
if (vfn instanceof KahanFunction && (op.aggOp.correctionLocation == CorrectionLocationType.LASTCOLUMN || op.aggOp.correctionLocation == CorrectionLocationType.LASTROW) && (ifn instanceof ReduceAll || ifn instanceof ReduceCol || ifn instanceof ReduceRow || ifn instanceof ReduceDiag)) {
if (vfn instanceof KahanPlus)
return AggType.KAHAN_SUM;
else if (vfn instanceof KahanPlusSq)
return AggType.KAHAN_SUM_SQ;
}
// mean
if (vfn instanceof Mean && (op.aggOp.correctionLocation == CorrectionLocationType.LASTTWOCOLUMNS || op.aggOp.correctionLocation == CorrectionLocationType.LASTTWOROWS) && (ifn instanceof ReduceAll || ifn instanceof ReduceCol || ifn instanceof ReduceRow)) {
return AggType.MEAN;
}
// variance
if (vfn instanceof CM && ((CM) vfn).getAggOpType() == AggregateOperationTypes.VARIANCE && (op.aggOp.correctionLocation == CorrectionLocationType.LASTFOURCOLUMNS || op.aggOp.correctionLocation == CorrectionLocationType.LASTFOURROWS) && (ifn instanceof ReduceAll || ifn instanceof ReduceCol || ifn instanceof ReduceRow)) {
return AggType.VAR;
}
// prod
if (vfn instanceof Multiply && ifn instanceof ReduceAll) {
return AggType.PROD;
}
// min / max
if (vfn instanceof Builtin && (ifn instanceof ReduceAll || ifn instanceof ReduceCol || ifn instanceof ReduceRow)) {
BuiltinCode bfcode = ((Builtin) vfn).bFunc;
switch(bfcode) {
case MAX:
return AggType.MAX;
case MIN:
return AggType.MIN;
case MAXINDEX:
return AggType.MAX_INDEX;
case MININDEX:
return AggType.MIN_INDEX;
// do nothing
default:
}
}
return AggType.INVALID;
}
use of org.apache.sysml.runtime.functionobjects.CM in project incubator-systemml by apache.
the class MatrixBlock method incrementalAggregate.
@Override
public void incrementalAggregate(AggregateOperator aggOp, MatrixValue newWithCorrection) {
// assert(aggOp.correctionExists);
MatrixBlock newWithCor = checkType(newWithCorrection);
KahanObject buffer = new KahanObject(0, 0);
if (aggOp.correctionLocation == CorrectionLocationType.LASTROW) {
if (aggOp.increOp.fn instanceof KahanPlus) {
LibMatrixAgg.aggregateBinaryMatrix(newWithCor, this, aggOp);
} else {
for (int r = 0; r < rlen - 1; r++) for (int c = 0; c < clen; c++) {
buffer._sum = this.quickGetValue(r, c);
buffer._correction = this.quickGetValue(r + 1, c);
buffer = (KahanObject) aggOp.increOp.fn.execute(buffer, newWithCor.quickGetValue(r, c), newWithCor.quickGetValue(r + 1, c));
quickSetValue(r, c, buffer._sum);
quickSetValue(r + 1, c, buffer._correction);
}
}
} else if (aggOp.correctionLocation == CorrectionLocationType.LASTCOLUMN) {
if (aggOp.increOp.fn instanceof Builtin && (((Builtin) (aggOp.increOp.fn)).bFunc == Builtin.BuiltinCode.MAXINDEX || ((Builtin) (aggOp.increOp.fn)).bFunc == Builtin.BuiltinCode.MININDEX)) {
// modified, the other needs to be changed to match.
for (int r = 0; r < rlen; r++) {
double currMaxValue = quickGetValue(r, 1);
long newMaxIndex = (long) newWithCor.quickGetValue(r, 0);
double newMaxValue = newWithCor.quickGetValue(r, 1);
double update = aggOp.increOp.fn.execute(newMaxValue, currMaxValue);
if (2.0 == update) {
// Return value of 2 ==> both values the same, break ties
// in favor of higher index.
long curMaxIndex = (long) quickGetValue(r, 0);
quickSetValue(r, 0, Math.max(curMaxIndex, newMaxIndex));
} else if (1.0 == update) {
// Return value of 1 ==> new value is better; use its index
quickSetValue(r, 0, newMaxIndex);
quickSetValue(r, 1, newMaxValue);
} else {
// Other return value ==> current answer is best
}
}
// *** END HACK ***
} else {
if (aggOp.increOp.fn instanceof KahanPlus) {
LibMatrixAgg.aggregateBinaryMatrix(newWithCor, this, aggOp);
} else {
for (int r = 0; r < rlen; r++) for (int c = 0; c < clen - 1; c++) {
buffer._sum = this.quickGetValue(r, c);
buffer._correction = this.quickGetValue(r, c + 1);
buffer = (KahanObject) aggOp.increOp.fn.execute(buffer, newWithCor.quickGetValue(r, c), newWithCor.quickGetValue(r, c + 1));
quickSetValue(r, c, buffer._sum);
quickSetValue(r, c + 1, buffer._correction);
}
}
}
} else if (aggOp.correctionLocation == CorrectionLocationType.LASTTWOROWS) {
double n, n2, mu2;
for (int r = 0; r < rlen - 2; r++) for (int c = 0; c < clen; c++) {
buffer._sum = this.quickGetValue(r, c);
n = this.quickGetValue(r + 1, c);
buffer._correction = this.quickGetValue(r + 2, c);
mu2 = newWithCor.quickGetValue(r, c);
n2 = newWithCor.quickGetValue(r + 1, c);
n = n + n2;
double toadd = (mu2 - buffer._sum) * n2 / n;
buffer = (KahanObject) aggOp.increOp.fn.execute(buffer, toadd);
quickSetValue(r, c, buffer._sum);
quickSetValue(r + 1, c, n);
quickSetValue(r + 2, c, buffer._correction);
}
} else if (aggOp.correctionLocation == CorrectionLocationType.LASTTWOCOLUMNS) {
double n, n2, mu2;
for (int r = 0; r < rlen; r++) for (int c = 0; c < clen - 2; c++) {
buffer._sum = this.quickGetValue(r, c);
n = this.quickGetValue(r, c + 1);
buffer._correction = this.quickGetValue(r, c + 2);
mu2 = newWithCor.quickGetValue(r, c);
n2 = newWithCor.quickGetValue(r, c + 1);
n = n + n2;
double toadd = (mu2 - buffer._sum) * n2 / n;
buffer = (KahanObject) aggOp.increOp.fn.execute(buffer, toadd);
quickSetValue(r, c, buffer._sum);
quickSetValue(r, c + 1, n);
quickSetValue(r, c + 2, buffer._correction);
}
} else if (aggOp.correctionLocation == CorrectionLocationType.LASTFOURROWS && aggOp.increOp.fn instanceof CM && ((CM) aggOp.increOp.fn).getAggOpType() == AggregateOperationTypes.VARIANCE) {
// create buffers to store results
CM_COV_Object cbuff_curr = new CM_COV_Object();
CM_COV_Object cbuff_part = new CM_COV_Object();
// perform incremental aggregation
for (int r = 0; r < rlen - 4; r++) for (int c = 0; c < clen; c++) {
// extract current values: { var | mean, count, m2 correction, mean correction }
// note: m2 = var * (n - 1)
// count
cbuff_curr.w = quickGetValue(r + 2, c);
// m2
cbuff_curr.m2._sum = quickGetValue(r, c) * (cbuff_curr.w - 1);
// mean
cbuff_curr.mean._sum = quickGetValue(r + 1, c);
cbuff_curr.m2._correction = quickGetValue(r + 3, c);
cbuff_curr.mean._correction = quickGetValue(r + 4, c);
// extract partial values: { var | mean, count, m2 correction, mean correction }
// note: m2 = var * (n - 1)
// count
cbuff_part.w = newWithCor.quickGetValue(r + 2, c);
// m2
cbuff_part.m2._sum = newWithCor.quickGetValue(r, c) * (cbuff_part.w - 1);
// mean
cbuff_part.mean._sum = newWithCor.quickGetValue(r + 1, c);
cbuff_part.m2._correction = newWithCor.quickGetValue(r + 3, c);
cbuff_part.mean._correction = newWithCor.quickGetValue(r + 4, c);
// calculate incremental aggregated variance
cbuff_curr = (CM_COV_Object) aggOp.increOp.fn.execute(cbuff_curr, cbuff_part);
// store updated values: { var | mean, count, m2 correction, mean correction }
double var = cbuff_curr.getRequiredResult(AggregateOperationTypes.VARIANCE);
quickSetValue(r, c, var);
// mean
quickSetValue(r + 1, c, cbuff_curr.mean._sum);
// count
quickSetValue(r + 2, c, cbuff_curr.w);
quickSetValue(r + 3, c, cbuff_curr.m2._correction);
quickSetValue(r + 4, c, cbuff_curr.mean._correction);
}
} else if (aggOp.correctionLocation == CorrectionLocationType.LASTFOURCOLUMNS && aggOp.increOp.fn instanceof CM && ((CM) aggOp.increOp.fn).getAggOpType() == AggregateOperationTypes.VARIANCE) {
// create buffers to store results
CM_COV_Object cbuff_curr = new CM_COV_Object();
CM_COV_Object cbuff_part = new CM_COV_Object();
// perform incremental aggregation
for (int r = 0; r < rlen; r++) for (int c = 0; c < clen - 4; c++) {
// extract current values: { var | mean, count, m2 correction, mean correction }
// note: m2 = var * (n - 1)
// count
cbuff_curr.w = quickGetValue(r, c + 2);
// m2
cbuff_curr.m2._sum = quickGetValue(r, c) * (cbuff_curr.w - 1);
// mean
cbuff_curr.mean._sum = quickGetValue(r, c + 1);
cbuff_curr.m2._correction = quickGetValue(r, c + 3);
cbuff_curr.mean._correction = quickGetValue(r, c + 4);
// extract partial values: { var | mean, count, m2 correction, mean correction }
// note: m2 = var * (n - 1)
// count
cbuff_part.w = newWithCor.quickGetValue(r, c + 2);
// m2
cbuff_part.m2._sum = newWithCor.quickGetValue(r, c) * (cbuff_part.w - 1);
// mean
cbuff_part.mean._sum = newWithCor.quickGetValue(r, c + 1);
cbuff_part.m2._correction = newWithCor.quickGetValue(r, c + 3);
cbuff_part.mean._correction = newWithCor.quickGetValue(r, c + 4);
// calculate incremental aggregated variance
cbuff_curr = (CM_COV_Object) aggOp.increOp.fn.execute(cbuff_curr, cbuff_part);
// store updated values: { var | mean, count, m2 correction, mean correction }
double var = cbuff_curr.getRequiredResult(AggregateOperationTypes.VARIANCE);
quickSetValue(r, c, var);
// mean
quickSetValue(r, c + 1, cbuff_curr.mean._sum);
// count
quickSetValue(r, c + 2, cbuff_curr.w);
quickSetValue(r, c + 3, cbuff_curr.m2._correction);
quickSetValue(r, c + 4, cbuff_curr.mean._correction);
}
} else
throw new DMLRuntimeException("unrecognized correctionLocation: " + aggOp.correctionLocation);
}
use of org.apache.sysml.runtime.functionobjects.CM in project incubator-systemml by apache.
the class CMCOVMRMapper method specialOperationsForActualMap.
@Override
protected void specialOperationsForActualMap(int index, OutputCollector<Writable, Writable> out, Reporter reporter) throws IOException {
// apply all instructions
processMapperInstructionsForMatrix(index);
for (byte tag : cmTags) {
CM lcmFn = cmFn.get(tag);
IndexedMatrixValue input = cachedValues.getFirst(tag);
if (input == null)
continue;
WeightedPair inputPair = (WeightedPair) input.getValue();
CM_N_COVCell cmValue = (CM_N_COVCell) cmNcovCache.getFirst(tag).getValue();
try {
lcmFn.execute(cmValue.getCM_N_COVObject(), inputPair.getValue(), inputPair.getWeight());
} catch (DMLRuntimeException e) {
throw new IOException(e);
}
}
for (byte tag : covTags) {
IndexedMatrixValue input = cachedValues.getFirst(tag);
if (input == null)
continue;
WeightedPair inputPair = (WeightedPair) input.getValue();
CM_N_COVCell comValue = (CM_N_COVCell) cmNcovCache.getFirst(tag).getValue();
try {
covFn.execute(comValue.getCM_N_COVObject(), inputPair.getValue(), inputPair.getOtherValue(), inputPair.getWeight());
} catch (DMLRuntimeException e) {
throw new IOException(e);
}
}
}
Aggregations